Changeset 48072

Timestamp:

06/17/2020 03:22:49 PM (4 years ago)

Author:

swissspidy

Message:

Sitemaps: Add XML sitemaps functionality to WordPress.

While web crawlers are able to discover pages from links within the site and from other sites, XML sitemaps supplement this approach by allowing crawlers to quickly and comprehensively identify all URLs included in the sitemap and learn other signals about those URLs using the associated metadata.

See ​https://make.wordpress.org/core/2020/06/10/merge-announcement-extensible-core-sitemaps/ for more details.

This feature exposes the sitemap index via /wp-sitemap.xml and exposes a variety of new filters and hooks for developers to modify the behavior. Users can disable sitemaps completely by turning off search engine visibility in WordPress admin.

This change also introduces a new esc_xml() function to escape strings for output in XML, as well as XML support to wp_kses_normalize_entities().

Props Adrian McShane, afragen, adamsilverstein, casiepa, flixos90, garrett-eclipse, joemcgill, kburgoine, kraftbj, milana_cap, pacifika, pbiron, pfefferle, Ruxandra Gradina, swissspidy, szepeviktor, tangrufus, tweetythierry.
Fixes #50117.
See #3670. See #19998.

Location:

trunk

Files:

• phpcs.xml.dist (modified) (1 diff)
• src/wp-includes/canonical.php (modified) (2 diffs)
• src/wp-includes/default-filters.php (modified) (1 diff)
• src/wp-includes/formatting.php (modified) (5 diffs)
• src/wp-includes/kses.php (modified) (4 diffs)
• src/wp-includes/sitemaps (added)
• src/wp-includes/sitemaps.php (added)
• src/wp-includes/sitemaps/class-wp-sitemaps-index.php (added)
• src/wp-includes/sitemaps/class-wp-sitemaps-provider.php (added)
• src/wp-includes/sitemaps/class-wp-sitemaps-registry.php (added)
• src/wp-includes/sitemaps/class-wp-sitemaps-renderer.php (added)
• src/wp-includes/sitemaps/class-wp-sitemaps-stylesheet.php (added)
• src/wp-includes/sitemaps/class-wp-sitemaps.php (added)
• src/wp-includes/sitemaps/providers (added)
• src/wp-includes/sitemaps/providers/class-wp-sitemaps-posts.php (added)
• src/wp-includes/sitemaps/providers/class-wp-sitemaps-taxonomies.php (added)
• src/wp-includes/sitemaps/providers/class-wp-sitemaps-users.php (added)
• src/wp-settings.php (modified) (1 diff)
• tests/phpunit/includes/bootstrap.php (modified) (2 diffs)
• tests/phpunit/includes/class-wp-sitemaps-empty-test-provider.php (added)
• tests/phpunit/includes/class-wp-sitemaps-test-provider.php (added)
• tests/phpunit/includes/normalize-xml.xsl (added)
• tests/phpunit/includes/testcase-xml.php (added)
• tests/phpunit/tests/canonical/sitemaps.php (added)
• tests/phpunit/tests/formatting/EscXml.php (added)
• tests/phpunit/tests/sitemaps (added)
• tests/phpunit/tests/sitemaps/functions.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-index.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-posts.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-registry.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-renderer.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-stylesheet.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-taxonomies.php (added)
• tests/phpunit/tests/sitemaps/sitemaps-users.php (added)
• tests/phpunit/tests/sitemaps/sitemaps.php (added)

trunk/phpcs.xml.dist

@@ -248 +248 @@
                 <element value="WP_Import_UnitTestCase"/>
                 <element value="Tests_Query_Conditionals"/>
+                <element value="WP_Test_XML_TestCase"/>
 
                 <!-- Mock classes. -->

trunk/src/wp-includes/canonical.php

@@ -508 +508 @@
             if ( ! empty( $addl_path ) ) {
                 $redirect['path'] = trailingslashit( $redirect['path'] ) . $addl_path;
+            }
+
+            // Remove trailing slash for sitemaps requests.
+            if ( ! empty( get_query_var( 'sitemap' ) ) ) {
+                $redirect['path'] = untrailingslashit( $redirect['path'] );
             }
 
@@ -652 +657 @@
     }
 
+    // Remove trailing slash for sitemaps requests.
+    if ( ! empty( get_query_var( 'sitemap' ) ) || ! empty( get_query_var( 'sitemap-stylesheet' ) ) ) {
+        $redirect['path'] = untrailingslashit( $redirect['path'] );
+    }
+
     // Strip multiple slashes out of the URL.
     if ( strpos( $redirect['path'], '//' ) > -1 ) {

trunk/src/wp-includes/default-filters.php

@@ -457 +457 @@
 add_action( 'parse_request', 'rest_api_loaded' );
 
+// Sitemaps actions.
+add_action( 'init', 'wp_sitemaps_get_server' );
+
 /**
  * Filters formerly mixed into wp-includes.

trunk/src/wp-includes/formatting.php

@@ -936 +936 @@
  *
  * @since 1.2.2
+ * @since 5.5.0 `$quote_style` also accepts '`ENT_XML1`.
  * @access private
  *
@@ -943 +944 @@
  * @param int|string   $quote_style   Optional. Converts double quotes if set to ENT_COMPAT,
  *                                    both single and double if set to ENT_QUOTES or none if set to ENT_NOQUOTES.
- *                                    Also compatible with old values; converting single quotes if set to 'single',
+ *                                    Converts single and double quotes, as well as converting HTML
+ *                                    named entities (that are not also XML named entities) to their
+ *                                    code points if set to ENT_XML1. Also compatible with old values;
+ *                                    converting single quotes if set to 'single',
  *                                    double if set to 'double' or both if otherwise set.
  *                                    Default is ENT_NOQUOTES.
@@ -965 +969 @@
     if ( empty( $quote_style ) ) {
         $quote_style = ENT_NOQUOTES;
-    } elseif ( ! in_array( $quote_style, array( 0, 2, 3, 'single', 'double' ), true ) ) {
+    } elseif ( ENT_XML1 === $quote_style ) {
+        $quote_style = ENT_QUOTES | ENT_XML1;
+    } elseif ( ! in_array( $quote_style, array( ENT_NOQUOTES, ENT_COMPAT, ENT_QUOTES, 'single', 'double' ), true ) ) {
         $quote_style = ENT_QUOTES;
     }
@@ -995 +1001 @@
         // Guarantee every &entity; is valid, convert &garbage; into &garbage;
         // This is required for PHP < 5.4.0 because ENT_HTML401 flag is unavailable.
-        $string = wp_kses_normalize_entities( $string );
+        $string = wp_kses_normalize_entities( $string, ( $quote_style & ENT_XML1 ) ? 'xml' : 'html' );
     }
 
@@ -4538 +4544 @@
 
 /**
+ * Escaping for XML blocks.
+ *
+ * @since 5.5.0
+ *
+ * @param string $text Text to escape.
+ * @return string Escaped text.
+ */
+function esc_xml( $text ) {
+    $safe_text = wp_check_invalid_utf8( $text );
+
+    $cdata_regex = '\<\!\[CDATA\[.*?\]\]\>';
+    $regex       = <<<EOF
+/
+    (?=.*?{$cdata_regex})                 # lookahead that will match anything followed by a CDATA Section
+    (?<non_cdata_followed_by_cdata>(.*?)) # the "anything" matched by the lookahead
+    (?<cdata>({$cdata_regex}))            # the CDATA Section matched by the lookahead
+
+|                                         # alternative
+
+    (?<non_cdata>(.*))                    # non-CDATA Section
+/sx
+EOF;
+
+    $safe_text = (string) preg_replace_callback(
+        $regex,
+        static function( $matches ) {
+            if ( ! $matches[0] ) {
+                return '';
+            }
+
+            if ( ! empty( $matches['non_cdata'] ) ) {
+                // escape HTML entities in the non-CDATA Section.
+                return _wp_specialchars( $matches['non_cdata'], ENT_XML1 );
+            }
+
+            // Return the CDATA Section unchanged, escape HTML entities in the rest.
+            return _wp_specialchars( $matches['non_cdata_followed_by_cdata'], ENT_XML1 ) . $matches['cdata'];
+        },
+        $safe_text
+    );
+
+    /**
+     * Filters a string cleaned and escaped for output in XML.
+     *
+     * Text passed to esc_xml() is stripped of invalid or special characters
+     * before output. HTML named character references are converted to their
+     * equivalent code points.
+     *
+     * @since 5.5.0
+     *
+     * @param string $safe_text The text after it has been escaped.
+     * @param string $text      The text prior to being escaped.
+     */
+    return apply_filters( 'esc_xml', $safe_text, $text );
+}
+
+/**
  * Escape an HTML tag name.
  *

trunk/src/wp-includes/kses.php

@@ -48 +48 @@
 // Ensure that these variables are added to the global namespace
 // (e.g. if using namespaces / autoload in the current PHP environment).
-global $allowedposttags, $allowedtags, $allowedentitynames;
+global $allowedposttags, $allowedtags, $allowedentitynames, $allowedxmlentitynames;
 
 if ( ! CUSTOM_TAGS ) {
@@ -705 +705 @@
     );
 
+    /**
+     * @var string[] $allowedxmlentitynames Array of KSES allowed XML entitity names.
+     * @since 5.5.0
+     */
+    $allowedxmlnamedentities = array(
+        'amp',
+        'lt',
+        'gt',
+        'apos',
+        'quot',
+    );
+
     $allowedposttags = array_map( '_wp_add_global_attributes', $allowedposttags );
 } else {
@@ -1746 +1758 @@
  * `AT&T`, `:` to `:`, `&#XYZZY;` to `&#XYZZY;` and so on.
  *
+ * When `$context` is set to 'xml', HTML entities are converted to their code points.  For
+ * example, `AT&T…&#XYZZY;` is converted to `AT&T…&#XYZZY;`.
+ *
  * @since 1.0.0
- *
- * @param string $string Content to normalize entities.
+ * @since 5.5.0 Added `$context` parameter.
+ *
+ * @param string $string  Content to normalize entities.
+ * @param string $context Context for normalization. Can be either 'html' or 'xml'.
+ *                        Default 'html'.
  * @return string Content with normalized entities.
  */
-function wp_kses_normalize_entities( $string ) {
+function wp_kses_normalize_entities( $string, $context = 'html' ) {
     // Disarm all entities by converting & to &
     $string = str_replace( '&', '&', $string );
 
     // Change back the allowed entities in our entity whitelist.
-    $string = preg_replace_callback( '/&([A-Za-z]{2,8}[0-9]{0,2});/', 'wp_kses_named_entities', $string );
+    if ( 'xml' === $context ) {
+        $string = preg_replace_callback( '/&([A-Za-z]{2,8}[0-9]{0,2});/', 'wp_kses_xml_named_entities', $string );
+    } else {
+        $string = preg_replace_callback( '/&([A-Za-z]{2,8}[0-9]{0,2});/', 'wp_kses_named_entities', $string );
+    }
     $string = preg_replace_callback( '/&#(0*[0-9]{1,7});/', 'wp_kses_normalize_entities2', $string );
     $string = preg_replace_callback( '/&#[Xx](0*[0-9A-Fa-f]{1,6});/', 'wp_kses_normalize_entities3', $string );
@@ -1785 +1807 @@
     $i = $matches[1];
     return ( ! in_array( $i, $allowedentitynames, true ) ) ? "&$i;" : "&$i;";
+}
+
+/**
+ * Callback for `wp_kses_normalize_entities()` regular expression.
+ *
+ * This function only accepts valid named entity references, which are finite,
+ * case-sensitive, and highly scrutinized by XML validators.  HTML named entity
+ * references are converted to their code points.
+ *
+ * @since 5.5.0
+ *
+ * @global array $allowedentitynames
+ * @global array $allowedxmlnamedentities
+ *
+ * @param array $matches preg_replace_callback() matches array.
+ * @return string Correctly encoded entity.
+ */
+function wp_kses_xml_named_entities( $matches ) {
+    global $allowedentitynames, $allowedxmlnamedentities;
+
+    if ( empty( $matches[1] ) ) {
+        return '';
+    }
+
+    $i = $matches[1];
+
+    if ( in_array( $i, $allowedxmlnamedentities, true ) ) {
+        return "&$i;";
+    } elseif ( in_array( $i, $allowedentitynames, true ) ) {
+        return html_entity_decode( "&$i;", ENT_HTML5 );
+    }
+
+    return "&$i;";
 }
 

trunk/src/wp-settings.php

@@ -264 +264 @@
 require ABSPATH . WPINC . '/rest-api/search/class-wp-rest-search-handler.php';
 require ABSPATH . WPINC . '/rest-api/search/class-wp-rest-post-search-handler.php';
+require ABSPATH . WPINC . '/sitemaps.php';
+require ABSPATH . WPINC . '/sitemaps/class-wp-sitemaps.php';
+require ABSPATH . WPINC . '/sitemaps/class-wp-sitemaps-index.php';
+require ABSPATH . WPINC . '/sitemaps/class-wp-sitemaps-provider.php';
+require ABSPATH . WPINC . '/sitemaps/class-wp-sitemaps-registry.php';
+require ABSPATH . WPINC . '/sitemaps/class-wp-sitemaps-renderer.php';
+require ABSPATH . WPINC . '/sitemaps/class-wp-sitemaps-stylesheet.php';
+require ABSPATH . WPINC . '/sitemaps/providers/class-wp-sitemaps-posts.php';
+require ABSPATH . WPINC . '/sitemaps/providers/class-wp-sitemaps-taxonomies.php';
+require ABSPATH . WPINC . '/sitemaps/providers/class-wp-sitemaps-users.php';
 require ABSPATH . WPINC . '/class-wp-block-type.php';
 require ABSPATH . WPINC . '/class-wp-block-styles-registry.php';

trunk/tests/phpunit/includes/bootstrap.php

@@ -154 +154 @@
 require __DIR__ . '/testcase-ajax.php';
 require __DIR__ . '/testcase-canonical.php';
+require __DIR__ . '/testcase-xml.php';
 require __DIR__ . '/exceptions.php';
 require __DIR__ . '/utils.php';
@@ -160 +161 @@
 require __DIR__ . '/class-wp-rest-test-configurable-controller.php';
 require __DIR__ . '/class-wp-fake-block-type.php';
+require __DIR__ . '/class-wp-sitemaps-test-provider.php';
+require __DIR__ . '/class-wp-sitemaps-empty-test-provider.php';
 
 /**