Changeset 45928

Timestamp:

09/02/2019 02:26:55 AM (5 years ago)

Author:

peterwilsoncc

Message:

#43590: Use robots meta tag to better discourage search engines.

This changes the "discourage search engines" option to output a noindex, nofollow robots meta tag. Disallow: / is removed from the robots.txt to allow search engines to discover they are requested not to index the site.

Disallowing search engines from accessing a site in the robots.txt file can result in search engines listing a site with a fragment (a listing without content).

Props donmhico, jonoaldersonwp.
Fixes #43590.

Location:

trunk

Files:

• src/wp-includes/functions.php (modified) (2 diffs)
• src/wp-includes/general-template.php (modified) (1 diff)
• tests/phpunit/tests/general/template.php (modified) (1 diff)

trunk/src/wp-includes/functions.php

@@ -1583 +1583 @@
  *
  * @since 2.1.0
+ * @since 5.3.0 Remove the "Disallow: /" output if search engine visiblity is
+ *              discouraged in favor of robots meta HTML tag in wp_no_robots().
  */
 function do_robots() {
@@ -1596 +1598 @@
     $output = "User-agent: *\n";
     $public = get_option( 'blog_public' );
-    if ( '0' == $public ) {
-        $output .= "Disallow: /\n";
-    } else {
-        $site_url = parse_url( site_url() );
-        $path     = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
-        $output  .= "Disallow: $path/wp-admin/\n";
-        $output  .= "Allow: $path/wp-admin/admin-ajax.php\n";
-    }
+
+    $site_url = parse_url( site_url() );
+    $path     = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
+    $output  .= "Disallow: $path/wp-admin/\n";
+    $output  .= "Allow: $path/wp-admin/admin-ajax.php\n";
 
     /**

trunk/src/wp-includes/general-template.php

@@ -2987 +2987 @@
  *
  * Outputs a noindex meta tag that tells web robots not to index the page content.
- * Typical usage is as a wp_head callback. add_action( 'wp_head', 'wp_no_robots' );
+ * Typical usage is as a {@see 'wp_head'} callback. add_action( 'wp_head', 'wp_no_robots' );
  *
  * @since 3.3.0
+ * @since 5.3.0 Echo "noindex,nofollow" if search engine visibility is discouraged.
  */
 function wp_no_robots() {
-    echo "<meta name='robots' content='noindex,follow' />\n";
+    if ( get_option( 'blog_public' ) ) {
+        echo "<meta name='robots' content='noindex,follow' />\n";
+        return;
+    }
+
+    echo "<meta name='robots' content='noindex,nofollow' />\n";
 }
 

trunk/tests/phpunit/tests/general/template.php

@@ -613 +613 @@
         $this->assertSame( $expected, $result );
     }
+
+    /**
+     * @ticket 43590
+     */
+    function test_wp_no_robots() {
+        // Simulate private site (search engines discouraged).
+        update_option( 'blog_public', '0' );
+        $actual_private = get_echo( 'wp_no_robots' );
+        $this->assertSame( "<meta name='robots' content='noindex,nofollow' />\n", $actual_private );
+
+        // Simulate public site.
+        update_option( 'blog_public', '1' );
+        $actual_public = get_echo( 'wp_no_robots' );
+        $this->assertSame( "<meta name='robots' content='noindex,follow' />\n", $actual_public );
+    }
 }