Ticket #21212: 21212-utf8mb4.7.diff

src/wp-admin/includes/schema.php

@@ -51 +51 @@
  slug varchar(200) NOT NULL default '',
  term_group bigint(10) NOT NULL default 0,
  PRIMARY KEY  (term_id),
- KEY slug (slug),
- KEY name (name)
+ KEY slug (slug(191)),
+ KEY name (name(191))
 ) $charset_collate;
 CREATE TABLE $wpdb->term_taxonomy (
  term_taxonomy_id bigint(20) unsigned NOT NULL auto_increment,
@@ -79 +79 @@
   meta_value longtext,
   PRIMARY KEY  (meta_id),
   KEY comment_id (comment_id),
-  KEY meta_key (meta_key)
+  KEY meta_key (meta_key(191))
 ) $charset_collate;
 CREATE TABLE $wpdb->comments (
   comment_ID bigint(20) unsigned NOT NULL auto_increment,
@@ -136 +136 @@
   meta_value longtext,
   PRIMARY KEY  (meta_id),
   KEY post_id (post_id),
-  KEY meta_key (meta_key)
+  KEY meta_key (meta_key(191))
 ) $charset_collate;
 CREATE TABLE $wpdb->posts (
   ID bigint(20) unsigned NOT NULL auto_increment,
@@ -163 +163 @@
   post_mime_type varchar(100) NOT NULL default '',
   comment_count bigint(20) NOT NULL default '0',
   PRIMARY KEY  (ID),
-  KEY post_name (post_name),
+  KEY post_name (post_name(191)),
   KEY type_status_date (post_type,post_status,post_date,ID),
   KEY post_parent (post_parent),
   KEY post_author (post_author)
@@ -213 +213 @@
   meta_value longtext,
   PRIMARY KEY  (umeta_id),
   KEY user_id (user_id),
-  KEY meta_key (meta_key)
+  KEY meta_key (meta_key(191))
 ) $charset_collate;\n";
 
         // Global tables
@@ -261 +261 @@
   domain varchar(200) NOT NULL default '',
   path varchar(100) NOT NULL default '',
   PRIMARY KEY  (id),
-  KEY domain (domain,path)
+  KEY domain (domain(140),path(51))
 ) $charset_collate;
 CREATE TABLE $wpdb->sitemeta (
   meta_id bigint(20) NOT NULL auto_increment,
@@ -269 +269 @@
   meta_key varchar(255) default NULL,
   meta_value longtext,
   PRIMARY KEY  (meta_id),
-  KEY meta_key (meta_key),
+  KEY meta_key (meta_key(191)),
   KEY site_id (site_id)
 ) $charset_collate;
 CREATE TABLE $wpdb->signups (
@@ -288 +288 @@
   KEY activation_key (activation_key),
   KEY user_email (user_email),
   KEY user_login_email (user_login,user_email),
-  KEY domain_path (domain,path)
+  KEY domain_path (domain(140),path(51))
 ) $charset_collate;";
 
         switch ( $scope ) {

src/wp-admin/includes/upgrade.php

@@ -519 +519 @@
         if ( $wp_current_db_version < 29630 )
                 upgrade_400();
 
+        if ( $wp_current_db_version < 30134 )
+                upgrade_420();
+
         maybe_disable_link_manager();
 
         maybe_disable_automattic_widgets();
@@ -1407 +1410 @@
 }
 
 /**
+ * Execute changes made in WordPress 4.2.0.
+ *
+ * @since 4.2.0
+ */
+function upgrade_420() {
+        global $wp_current_db_version, $wpdb;
+
+        if ( $wp_current_db_version < 30134 && $wpdb->charset === 'utf8mb4' ) {
+                if ( is_multisite() ) {
+                        $tables = $wpdb->tables;
+                } else {
+                        $tables = array_merge( $wpdb->tables, $wpdb->global_tables );
+                }
+
+                foreach ( $tables as $table ) {
+                        $results = $wpdb->get_results( "SHOW FULL COLUMNS FROM `{$wpdb->$table}`" );
+                        if ( ! $results ) {
+                                continue;
+                        }
+
+                        $has_utf8 = false;
+                        foreach ( $results as $column ) {
+                                if ( $column->Collation ) {
+                                        if ( 'utf8' === $column->Collation ) {
+                                                $has_utf8 = true;
+                                        } else {
+                                                // Don't upgrade tables that have non-utf8 columns
+                                                continue 2;
+                                        }
+                                }
+                        }
+
+                        if ( ! $has_utf8 ) {
+                                // Don't bother upgrading tables that don't have utf8 columns
+                                continue;
+                        }
+
+                        $wpdb->query( "ALTER TABLE {$wpdb->$table} CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" );
+                }
+        }
+}
+
+/**
  * Executes network-level upgrade routines.
  *
  * @since 3.0.0
@@ -1502 +1548 @@
                         update_site_option( 'illegal_names', $illegal_names );
                 }
         }
+
+        // 4.2
+        if ( $wp_current_db_version < 30134 && $wpdb->charset === 'utf8mb4') {
+                $wpdb->query( "ALTER TABLE $wpdb->site DROP INDEX domain, ADD INDEX domain(domain(140),path(51))" );
+                $wpdb->query( "ALTER TABLE $wpdb->sitemeta DROP INDEX meta_key, ADD INDEX meta_key(meta_key(191))" );
+                $wpdb->query( "ALTER TABLE $wpdb->signups DROP INDEX domain, ADD INDEX domain(domain(140),path(51))" );
+
+                $tables = array_merge( $wpdb->ms_global_tables, $wpdb->global_tables );
+
+                foreach ( $tables as $table ) {
+                        $results = $wpdb->get_results( "SHOW FULL COLUMNS FROM `{$wpdb->$table}`" );
+                        if ( ! $results ) {
+                                continue;
+                        }
+
+                        $has_utf8 = false;
+                        foreach ( $results as $column ) {
+                                if ( $column->Collation ) {
+                                        if ( 'utf8' === $column->Collation ) {
+                                                $has_utf8 = true;
+                                        } else {
+                                                // Don't upgrade tables that have non-utf8 columns
+                                                continue 2;
+                                        }
+                                }
+                        }
+
+                        if ( ! $has_utf8 ) {
+                                // Don't bother upgrading tables that don't have utf8 columns
+                                continue;
+                        }
+
+                        $wpdb->query( "ALTER TABLE {$wpdb->$table} CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" );
+                }
+        }
 }
 
 //
@@ -2284 +2365 @@
                 // dbDelta() can recreate but can't drop the index.
                 $wpdb->query( "ALTER TABLE $wpdb->terms DROP INDEX slug" );
         }
+
+        if ( $wp_current_db_version < 30134 ) {
+                // We need to alter some indices
+                $wpdb->query( "ALTER TABLE $wpdb->usermeta DROP INDEX meta_key, ADD INDEX meta_key(meta_key(191))" );
+                $wpdb->query( "ALTER TABLE $wpdb->terms DROP INDEX slug, ADD INDEX slug(slug(191))" );
+                $wpdb->query( "ALTER TABLE $wpdb->terms DROP INDEX name, ADD INDEX name(name(191))" );
+                $wpdb->query( "ALTER TABLE $wpdb->commentmeta DROP INDEX meta_key, ADD INDEX meta_key(meta_key(191))" );
+                $wpdb->query( "ALTER TABLE $wpdb->postmeta DROP INDEX meta_key, ADD INDEX meta_key(meta_key(191))" );
+                $wpdb->query( "ALTER TABLE $wpdb->posts DROP INDEX post_name, ADD INDEX post_name(post_name(191))" );
+        }
 }
 
 /**

src/wp-admin/setup-config.php

@@ -280 +280 @@
                         case 'DB_HOST'     :
                                 $config_file[ $line_num ] = "define('" . $constant . "'," . $padding . "'" . addcslashes( constant( $constant ), "\\'" ) . "');\r\n";
                                 break;
+                        case 'DB_CHARSET'  :
+                                if ( 'utf8mb4' === $wpdb->charset || ( ! $wpdb->charset && $wpdb->has_cap( 'utf8mb4' ) ) ) {
+                                        $config_file[ $line_num ] = "define('" . $constant . "'," . $padding . "'utf8mb4');\r\n";
+                                }
+                                break;
                         case 'AUTH_KEY'         :
                         case 'SECURE_AUTH_KEY'  :
                         case 'LOGGED_IN_KEY'    :

src/wp-includes/version.php

@@ -11 +11 @@
  *
  * @global int $wp_db_version
  */
-$wp_db_version = 30133;
+$wp_db_version = 30134;
 
 /**
  * Holds the TinyMCE version

src/wp-includes/wp-db.php

@@ -624 +624 @@
                         }
                 }
 
-                $this->init_charset();
-
                 $this->dbuser = $dbuser;
                 $this->dbpassword = $dbpassword;
                 $this->dbname = $dbname;
@@ -717 +715 @@
         public function init_charset() {
                 if ( function_exists('is_multisite') && is_multisite() ) {
                         $this->charset = 'utf8';
-                        if ( defined( 'DB_COLLATE' ) && DB_COLLATE )
+                        if ( defined( 'DB_COLLATE' ) && DB_COLLATE ) {
                                 $this->collate = DB_COLLATE;
-                        else
+                        } else {
                                 $this->collate = 'utf8_general_ci';
+                        }
                 } elseif ( defined( 'DB_COLLATE' ) ) {
                         $this->collate = DB_COLLATE;
                 }
 
-                if ( defined( 'DB_CHARSET' ) )
+                if ( defined( 'DB_CHARSET' ) ) {
                         $this->charset = DB_CHARSET;
+                }
+
+                if ( ( $this->use_mysqli && ! ( $this->dbh instanceof mysqli ) )
+                  || ( empty( $this->dbh ) || ! ( $this->dbh instanceof mysqli ) ) ) {
+                        return;
+                }
+
+                if ( 'utf8' === $this->charset && $this->has_cap( 'utf8mb4' ) ) {
+                        $this->charset = 'utf8mb4';
+                }
+
+                if ( 'utf8mb4' === $this->charset && ( ! $this->collate || stripos( $this->collate, 'utf8_' ) === 0 ) ) {
+                        $this->collate = 'utf8mb4_unicode_ci';
+                }
         }
 
         /**
@@ -1477 +1490 @@
                         return false;
                 } elseif ( $this->dbh ) {
                         $this->has_connected = true;
+
+                        $this->init_charset();
                         $this->set_charset( $this->dbh );
+
                         $this->ready = true;
                         $this->set_sql_mode();
                         $this->select( $this->dbname, $this->dbh );
@@ -2249 +2265 @@
          * Retrieves the character set for the given column.
          *
          * @since 4.2.0
-         * @access protected
+         * @access public
          *
          * @param string $table  Table name.
          * @param string $column Column name.
@@ -2256 +2272 @@
          * @return mixed Column character set as a string. False if the column has no
          *               character set. {@see WP_Error} object if there was an error.
          */
-        protected function get_col_charset( $table, $column ) {
+        public function get_col_charset( $table, $column ) {
                 $tablekey = strtolower( $table );
                 $columnkey = strtolower( $column );
 
@@ -2356 +2372 @@
                         'gb2312'  => 'EUC-CN',
                         'ujis'    => 'EUC-JP',
                         'utf32'   => 'UTF-32',
-                        'utf8mb4' => 'UTF-8',
                 );
 
                 $supported_charsets = array();
@@ -2391 +2406 @@
                                 }
                         }
 
-                        // utf8(mb3) can be handled by regex, which is a bunch faster than a DB lookup.
-                        if ( 'utf8' === $charset || 'utf8mb3' === $charset ) {
+                        // utf8 can be handled by regex, which is a bunch faster than a DB lookup.
+                        if ( 'utf8' === $charset || 'utf8mb3' === $charset || 'utf8mb4' === $charset ) {
                                 $regex = '/
                                         (
                                                 (?: [\x00-\x7F]                  # single-byte sequences   0xxxxxxx
@@ -2400 +2415 @@
                                                 |   \xE0[\xA0-\xBF][\x80-\xBF]   # triple-byte sequences   1110xxxx 10xxxxxx * 2
                                                 |   [\xE1-\xEC][\x80-\xBF]{2}
                                                 |   \xED[\x80-\x9F][\x80-\xBF]
-                                                |   [\xEE-\xEF][\x80-\xBF]{2}
-                                                ){1,50}                          # ...one or more times
+                                                |   [\xEE-\xEF][\x80-\xBF]{2}';
+
+                                if ( 'utf8mb4' === $charset) {
+                                        $regex .= '
+                                                |    \xF0[\x90-\xBF][\x80-\xBF]{2} # four-byte sequences   11110xxx 10xxxxxx * 3
+                                                |    [\xF1-\xF3][\x80-\xBF]{3}
+                                                |    \xF4[\x80-\x8F][\x80-\xBF]{2}
+                                        ';
+                                }
+
+                                $regex .= '){1,50}                          # ...one or more times
                                         )
                                         | .                                  # anything else
                                         /x';

tests/phpunit/tests/db/charset.php

@@ -130 +130 @@
         }
 
         /**
-         * @ ticket 21212
+         * @ticket 21212
          */
         function test_process_fields_failure() {
                 global $wpdb;
-                $data = array( 'post_content' => "H€llo\xf0\x9f\x98\x88World¢" );
+                // \xf0\xff\xff\xff is invalid in utf8 and utf8mb4
+                $data = array( 'post_content' => "H€llo\xf0\xff\xff\xffWorld¢" );
                 $this->assertFalse( self::$_wpdb->process_fields( $wpdb->posts, $data, null ) );
         }
 
@@ -436 +437 @@
          */
         function test_invalid_characters_in_query() {
                 global $wpdb;
-                $this->assertFalse( $wpdb->query( "INSERT INTO {$wpdb->posts} (post_content) VALUES ('foo\xf0\x9f\x98\x88bar')" ) );
+                $this->assertFalse( $wpdb->query( "INSERT INTO {$wpdb->posts} (post_content) VALUES ('foo\xf0\xff\xff\xffbar')" ) );
         }
 }