WordPress.org

Make WordPress Core

Changeset 45928


Ignore:
Timestamp:
09/02/2019 02:26:55 AM (3 months ago)
Author:
peterwilsoncc
Message:

#43590: Use robots meta tag to better discourage search engines.

This changes the "discourage search engines" option to output a noindex, nofollow robots meta tag. Disallow: / is removed from the robots.txt to allow search engines to discover they are requested not to index the site.

Disallowing search engines from accessing a site in the robots.txt file can result in search engines listing a site with a fragment (a listing without content).

Props donmhico, jonoaldersonwp.
Fixes #43590.

Location:
trunk
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/wp-includes/functions.php

    r45926 r45928  
    15831583 *
    15841584 * @since 2.1.0
     1585 * @since 5.3.0 Remove the "Disallow: /" output if search engine visiblity is
     1586 *              discouraged in favor of robots meta HTML tag in wp_no_robots().
    15851587 */
    15861588function do_robots() {
     
    15961598    $output = "User-agent: *\n";
    15971599    $public = get_option( 'blog_public' );
    1598     if ( '0' == $public ) {
    1599         $output .= "Disallow: /\n";
    1600     } else {
    1601         $site_url = parse_url( site_url() );
    1602         $path     = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
    1603         $output  .= "Disallow: $path/wp-admin/\n";
    1604         $output  .= "Allow: $path/wp-admin/admin-ajax.php\n";
    1605     }
     1600
     1601    $site_url = parse_url( site_url() );
     1602    $path     = ( ! empty( $site_url['path'] ) ) ? $site_url['path'] : '';
     1603    $output  .= "Disallow: $path/wp-admin/\n";
     1604    $output  .= "Allow: $path/wp-admin/admin-ajax.php\n";
    16061605
    16071606    /**
  • trunk/src/wp-includes/general-template.php

    r45926 r45928  
    29872987 *
    29882988 * Outputs a noindex meta tag that tells web robots not to index the page content.
    2989  * Typical usage is as a wp_head callback. add_action( 'wp_head', 'wp_no_robots' );
     2989 * Typical usage is as a {@see 'wp_head'} callback. add_action( 'wp_head', 'wp_no_robots' );
    29902990 *
    29912991 * @since 3.3.0
     2992 * @since 5.3.0 Echo "noindex,nofollow" if search engine visibility is discouraged.
    29922993 */
    29932994function wp_no_robots() {
    2994     echo "<meta name='robots' content='noindex,follow' />\n";
     2995    if ( get_option( 'blog_public' ) ) {
     2996        echo "<meta name='robots' content='noindex,follow' />\n";
     2997        return;
     2998    }
     2999
     3000    echo "<meta name='robots' content='noindex,nofollow' />\n";
    29953001}
    29963002
  • trunk/tests/phpunit/tests/general/template.php

    r45028 r45928  
    613613        $this->assertSame( $expected, $result );
    614614    }
     615
     616    /**
     617     * @ticket 43590
     618     */
     619    function test_wp_no_robots() {
     620        // Simulate private site (search engines discouraged).
     621        update_option( 'blog_public', '0' );
     622        $actual_private = get_echo( 'wp_no_robots' );
     623        $this->assertSame( "<meta name='robots' content='noindex,nofollow' />\n", $actual_private );
     624
     625        // Simulate public site.
     626        update_option( 'blog_public', '1' );
     627        $actual_public = get_echo( 'wp_no_robots' );
     628        $this->assertSame( "<meta name='robots' content='noindex,follow' />\n", $actual_public );
     629    }
    615630}
Note: See TracChangeset for help on using the changeset viewer.