Changeset 49992

Timestamp:

01/21/2021 01:35:16 AM (4 years ago)

Author:

flixos90

Message:

Robots: Introduce Robots API.

This changeset introduces a filter-based Robots API, providing central control over the robots meta tag.

• Introduces wp_robots() function which should be called anywhere a robots meta tag should be included.
• Introduces wp_robots filter which allows adding or modifying directives for the robots meta tag. The wp_robots() function is entirely filter-based, i.e. if no filter is added to wp_robots, no directives will be present, and therefore the entire robots meta tag will be omitted.
• Introduces the following wp_robots filter functions which replace similar existing functions that were manually rendering a robots meta tag:
  • wp_robots_noindex() replaces noindex(), which has been deprecated.
  • wp_robots_no_robots() replaces wp_no_robots(), which has been deprecated.
  • wp_robots_sensitive_page() replaces wp_sensitive_page_meta(), which has been deprecated. Its rendering of the referrer meta tag has been moved to another new function wp_strict_cross_origin_referrer().

Migration to the new functions is straightforward. For example, a call to add_action( 'wp_head', 'wp_no_robots' ) should be replaced with add_filter( 'wp_robots', 'wp_robots_no_robots' ).

Plugins and themes that render their own robots meta tags are encouraged to switch to rely on the wp_robots filter in order to use the central management layer now provided by WordPress core.

Props adamsilverstein, flixos90, timothyblynjacobs, westonruter.
See #51511.

Location:

trunk

Files:

• src/wp-activate.php (modified) (1 diff)
• src/wp-includes/class-wp-customize-manager.php (modified) (1 diff)
• src/wp-includes/default-filters.php (modified) (5 diffs)
• src/wp-includes/deprecated.php (modified) (1 diff)
• src/wp-includes/embed.php (modified) (1 diff)
• src/wp-includes/functions.php (modified) (2 diffs)
• src/wp-includes/general-template.php (modified) (1 diff)
• src/wp-includes/robots-template.php (added)
• src/wp-login.php (modified) (1 diff)
• src/wp-settings.php (modified) (1 diff)
• src/wp-signup.php (modified) (1 diff)
• tests/phpunit/tests/customize/manager.php (modified) (1 diff)
• tests/phpunit/tests/general/template.php (modified) (1 diff)
• tests/phpunit/tests/robots.php (added)

trunk/src/wp-activate.php

@@ -115 +115 @@
 }
 add_action( 'wp_head', 'wpmu_activate_stylesheet' );
-add_action( 'wp_head', 'wp_sensitive_page_meta' );
+add_action( 'wp_head', 'wp_strict_cross_origin_referrer' );
+add_filter( 'wp_robots', 'wp_robots_sensitive_page' );
 
 get_header( 'wp-activate' );

trunk/src/wp-includes/class-wp-customize-manager.php

@@ -1901 +1901 @@
             header( 'X-Robots: noindex, nofollow, noarchive' );
         }
-        add_action( 'wp_head', 'wp_no_robots' );
+        add_filter( 'wp_robots', 'wp_robots_no_robots' );
         add_filter( 'wp_headers', array( $this, 'filter_iframe_security_headers' ) );
 

trunk/src/wp-includes/default-filters.php

@@ -231 +231 @@
 // Email filters.
 add_filter( 'wp_mail', 'wp_staticize_emoji_for_email' );
+
+// Robots filters.
+add_filter( 'wp_robots', 'wp_robots_noindex' );
 
 // Mark site as no longer fresh.
@@ -292 +295 @@
 add_action( 'wp_head', 'locale_stylesheet' );
 add_action( 'publish_future_post', 'check_and_publish_future_post', 10, 1 );
-add_action( 'wp_head', 'noindex', 1 );
+add_action( 'wp_head', 'wp_robots', 1 );
 add_action( 'wp_head', 'print_emoji_detection_script', 7 );
 add_action( 'wp_head', 'wp_print_styles', 8 );
@@ -312 +315 @@
 
 if ( isset( $_GET['replytocom'] ) ) {
-    add_action( 'wp_head', 'wp_no_robots' );
+    add_filter( 'wp_robots', 'wp_robots_no_robots' );
 }
 
 // Login actions.
+add_action( 'login_head', 'wp_robots', 1 );
 add_filter( 'login_head', 'wp_resource_hints', 8 );
 add_action( 'login_head', 'wp_print_head_scripts', 9 );
@@ -586 +590 @@
 add_action( 'embed_head', 'wp_print_head_scripts', 20 );
 add_action( 'embed_head', 'wp_print_styles', 20 );
-add_action( 'embed_head', 'wp_no_robots' );
+add_action( 'embed_head', 'wp_robots' );
 add_action( 'embed_head', 'rel_canonical' );
 add_action( 'embed_head', 'locale_stylesheet', 30 );
@@ -597 +601 @@
 add_action( 'embed_footer', 'wp_print_footer_scripts', 20 );
 
+add_filter( 'wp_robots', 'wp_embed_no_robots' );
 add_filter( 'excerpt_more', 'wp_embed_excerpt_more', 20 );
 add_filter( 'the_excerpt_embed', 'wptexturize' );

trunk/src/wp-includes/deprecated.php

@@ -4136 +4136 @@
     return is_string( $value ) ? addslashes( $value ) : $value;
 }
+
+/**
+ * Displays a noindex meta tag if required by the blog configuration.
+ *
+ * If a blog is marked as not being public then the noindex meta tag will be
+ * output to tell web robots not to index the page content. Add this to the
+ * {@see 'wp_head'} action.
+ *
+ * Typical usage is as a {@see 'wp_head'} callback:
+ *
+ *     add_action( 'wp_head', 'noindex' );
+ *
+ * @see wp_no_robots()
+ *
+ * @since 2.1.0
+ * @deprecated 5.7.0 Use wp_robots_noindex() instead on 'wp_robots' filter.
+ */
+function noindex() {
+    _deprecated_function( __FUNCTION__, '5.7.0', 'wp_robots_noindex()' );
+
+    // If the blog is not public, tell robots to go away.
+    if ( '0' == get_option( 'blog_public' ) ) {
+        wp_no_robots();
+    }
+}
+
+/**
+ * Display a noindex meta tag.
+ *
+ * Outputs a noindex meta tag that tells web robots not to index the page content.
+ * Typical usage is as a {@see 'wp_head'} callback. add_action( 'wp_head', 'wp_no_robots' );
+ *
+ * @since 3.3.0
+ * @since 5.3.0 Echo "noindex,nofollow" if search engine visibility is discouraged.
+ * @deprecated 5.7.0 Use wp_robots_no_robots() instead on 'wp_robots' filter.
+ */
+function wp_no_robots() {
+    _deprecated_function( __FUNCTION__, '5.7.0', 'wp_robots_no_robots()' );
+
+    if ( get_option( 'blog_public' ) ) {
+        echo "<meta name='robots' content='noindex,follow' />\n";
+        return;
+    }
+
+    echo "<meta name='robots' content='noindex,nofollow' />\n";
+}
+
+/**
+ * Display a noindex,noarchive meta tag and referrer origin-when-cross-origin meta tag.
+ *
+ * Outputs a noindex,noarchive meta tag that tells web robots not to index or cache the page content.
+ * Outputs a referrer origin-when-cross-origin meta tag that tells the browser not to send the full
+ * url as a referrer to other sites when cross-origin assets are loaded.
+ *
+ * Typical usage is as a wp_head callback. add_action( 'wp_head', 'wp_sensitive_page_meta' );
+ *
+ * @since 5.0.1
+ * @deprecated 5.7.0 Use wp_robots_sensitive_page() instead on 'wp_robots' filter
+ *                   and wp_strict_cross_origin_referrer() on 'wp_head' action.
+ */
+function wp_sensitive_page_meta() {
+    _deprecated_function( __FUNCTION__, '5.7.0', 'wp_robots_sensitive_page()' );
+
+    ?>
+    <meta name='robots' content='noindex,noarchive' />
+    <?php
+    wp_strict_cross_origin_referrer();
+}

trunk/src/wp-includes/embed.php

@@ -1244 +1244 @@
     return $result;
 }
+
+/**
+ * Adds noindex to the robots meta tag for embeds.
+ *
+ * Typical usage is as a {@see 'wp_robots'} callback:
+ *
+ *     add_filter( 'wp_robots', 'wp_embed_no_robots' );
+ *
+ * @since 5.7.0
+ *
+ * @param array $robots Associative array of robots directives.
+ * @return array Filtered robots directives.
+ */
+function wp_embed_no_robots( array $robots ) {
+    if ( ! is_embed() ) {
+        return $robots;
+    }
+
+    return wp_robots_no_robots( $robots );
+}

trunk/src/wp-includes/functions.php

@@ -1641 +1641 @@
  * @since 2.1.0
  * @since 5.3.0 Remove the "Disallow: /" output if search engine visiblity is
- *              discouraged in favor of robots meta HTML tag in wp_no_robots().
+ *              discouraged in favor of robots meta HTML tag via wp_robots_no_robots()
+ *              filter callback.
  */
 function do_robots() {
@@ -3492 +3493 @@
     <meta name="viewport" content="width=device-width">
         <?php
-        if ( function_exists( 'wp_no_robots' ) ) {
-            wp_no_robots();
+        if ( function_exists( 'wp_robots' ) && function_exists( 'wp_robots_no_robots' ) && function_exists( 'add_filter' ) ) {
+            add_filter( 'wp_robots', 'wp_robots_no_robots' );
+            wp_robots();
         }
         ?>

trunk/src/wp-includes/general-template.php

@@ -3192 +3192 @@
 
 /**
- * Displays a noindex meta tag if required by the blog configuration.
- *
- * If a blog is marked as not being public then the noindex meta tag will be
- * output to tell web robots not to index the page content. Add this to the
- * {@see 'wp_head'} action.
- *
- * Typical usage is as a {@see 'wp_head'} callback:
- *
- *     add_action( 'wp_head', 'noindex' );
- *
- * @see wp_no_robots()
- *
- * @since 2.1.0
- */
-function noindex() {
-    // If the blog is not public, tell robots to go away.
-    if ( '0' == get_option( 'blog_public' ) ) {
-        wp_no_robots();
-    }
-}
-
-/**
- * Display a noindex meta tag.
- *
- * Outputs a noindex meta tag that tells web robots not to index the page content.
- * Typical usage is as a {@see 'wp_head'} callback. add_action( 'wp_head', 'wp_no_robots' );
- *
- * @since 3.3.0
- * @since 5.3.0 Echo "noindex,nofollow" if search engine visibility is discouraged.
- */
-function wp_no_robots() {
-    if ( get_option( 'blog_public' ) ) {
-        echo "<meta name='robots' content='noindex,follow' />\n";
-        return;
-    }
-
-    echo "<meta name='robots' content='noindex,nofollow' />\n";
-}
-
-/**
- * Display a noindex,noarchive meta tag and referrer origin-when-cross-origin meta tag.
- *
- * Outputs a noindex,noarchive meta tag that tells web robots not to index or cache the page content.
+ * Displays a referrer strict-origin-when-cross-origin meta tag.
+ *
  * Outputs a referrer origin-when-cross-origin meta tag that tells the browser not to send the full
  * url as a referrer to other sites when cross-origin assets are loaded.
  *
- * Typical usage is as a wp_head callback. add_action( 'wp_head', 'wp_sensitive_page_meta' );
- *
- * @since 5.0.1
- */
-function wp_sensitive_page_meta() {
+ * Typical usage is as a wp_head callback. add_action( 'wp_head', 'wp_strict_cross_origin_referrer' );
+ *
+ * @since 5.7.0
+ */
+function wp_strict_cross_origin_referrer() {
     ?>
-    <meta name='robots' content='noindex,noarchive' />
     <meta name='referrer' content='strict-origin-when-cross-origin' />
     <?php

trunk/src/wp-login.php

@@ -43 +43 @@
 
     // Don't index any of these forms.
-    add_action( 'login_head', 'wp_sensitive_page_meta' );
+    add_filter( 'wp_robots', 'wp_robots_sensitive_page' );
+    add_action( 'login_head', 'wp_strict_cross_origin_referrer' );
 
     add_action( 'login_head', 'wp_login_viewport_meta' );

trunk/src/wp-settings.php

@@ -182 +182 @@
 require ABSPATH . WPINC . '/link-template.php';
 require ABSPATH . WPINC . '/author-template.php';
+require ABSPATH . WPINC . '/robots-template.php';
 require ABSPATH . WPINC . '/post.php';
 require ABSPATH . WPINC . '/class-walker-page.php';

trunk/src/wp-signup.php

@@ -4 +4 @@
 require __DIR__ . '/wp-load.php';
 
-add_action( 'wp_head', 'wp_no_robots' );
+add_filter( 'wp_robots', 'wp_robots_no_robots' );
 
 require __DIR__ . '/wp-blog-header.php';

trunk/tests/phpunit/tests/customize/manager.php

@@ -894 +894 @@
         $this->assertSame( $did_action_customize_preview_init + 1, did_action( 'customize_preview_init' ) );
 
-        $this->assertSame( 10, has_action( 'wp_head', 'wp_no_robots' ) );
+        $this->assertSame( 10, has_filter( 'wp_robots', 'wp_robots_no_robots' ) );
         $this->assertSame( 10, has_action( 'wp_head', array( $wp_customize, 'remove_frameless_preview_messenger_channel' ) ) );
         $this->assertSame( 10, has_filter( 'wp_headers', array( $wp_customize, 'filter_iframe_security_headers' ) ) );

trunk/tests/phpunit/tests/general/template.php

@@ -473 +473 @@
 
         $this->assertSame( $expected, $result );
-    }
-
-    /**
-     * @ticket 43590
-     */
-    function test_wp_no_robots() {
-        // Simulate private site (search engines discouraged).
-        update_option( 'blog_public', '0' );
-        $actual_private = get_echo( 'wp_no_robots' );
-        $this->assertSame( "<meta name='robots' content='noindex,nofollow' />\n", $actual_private );
-
-        // Simulate public site.
-        update_option( 'blog_public', '1' );
-        $actual_public = get_echo( 'wp_no_robots' );
-        $this->assertSame( "<meta name='robots' content='noindex,follow' />\n", $actual_public );
     }