Changeset 60949

Timestamp:

10/16/2025 08:58:26 PM (4 months ago)

Author:

dmsnell

Message:

Charset: Rely on new UTF-8 pipeline for mb_strlen() fallback.

The existing polyfill for mb_strlen() contains a number of issues leaving plenty of opportunity for improvement. Specifically, the following are all deficiencies: it relies on Unicode PCRE support, assumes input strings are valid UTF-8, splits input strings into an array of character to count them (1,000 at a time, iterating until complete), and entirely gives up when the Unicode support is missing.

This patch provides an updated polyfill which will reliably count code points in a UTF-8 string, even in the presence of sequences of invalid bytes. It scans through the input with zero allocations. Additionally, the underlying fallback extends the behavior of mb_strlen() to provide character counts for substrings within a larger input without extracting the substring (it can counts characters within a byte offset and length of a larger string).

This change improves the reliability of UTF-8 string length calculations and removes behavioral variability based on the runtime system.

Developed in ​https://github.com/WordPress/wordpress-develop/pull/9828
Discussed in https://core.trac.wordpress.org/ticket/63863

See #63863.

Location:

trunk

Files:

• src/wp-includes/compat-utf8.php (modified) (1 diff)
• src/wp-includes/compat.php (modified) (2 diffs)
• tests/phpunit/tests/compat/mbStrlen.php (modified) (2 diffs)
• tests/phpunit/tests/compat/wpCodePointCount.php (added)

trunk/src/wp-includes/compat-utf8.php

@@ -292 +292 @@
     return $scrubbed;
 }
+
+/**
+ * Returns how many code points are found in the given UTF-8 string.
+ *
+ * Invalid spans of bytes count as a single code point according
+ * to the maximal subpart rule. This function is a fallback method
+ * for calling `mb_strlen( $text, 'UTF-8' )`.
+ *
+ * When negative values are provided for the byte offsets or length,
+ * this will always report zero code points.
+ *
+ * Example:
+ *
+ *     4  === _wp_utf8_codepoint_count( 'text' );
+ *
+ *     // Groups are 'test', "\x90" as '�', 'wp', "\xE2\x80" as '�', "\xC0" as '�', and 'test'.
+ *     13 === _wp_utf8_codepoint_count( "test\x90wp\xE2\x80\xC0test" );
+ *
+ * @since 6.9.0
+ * @access private
+ *
+ * @param string $text            Count code points in this string.
+ * @param ?int   $byte_offset     Start counting after this many bytes in `$text`. Must be positive.
+ * @param ?int   $max_byte_length Optional. Stop counting after having scanned past this many bytes.
+ *                                Default is to scan until the end of the string. Must be positive.
+ * @return int How many code points were found.
+ */
+function _wp_utf8_codepoint_count( string $text, ?int $byte_offset = 0, ?int $max_byte_length = PHP_INT_MAX ): int {
+    if ( $byte_offset < 0 ) {
+        return 0;
+    }
+
+    $count           = 0;
+    $at              = $byte_offset;
+    $end             = strlen( $text );
+    $invalid_length  = 0;
+    $max_byte_length = min( $end - $at, $max_byte_length );
+
+    while ( $at < $end && ( $at - $byte_offset ) < $max_byte_length ) {
+        $count += _wp_scan_utf8( $text, $at, $invalid_length, $max_byte_length - ( $at - $byte_offset ) );
+        $count += $invalid_length > 0 ? 1 : 0;
+        $at    += $invalid_length;
+    }
+
+    return $count;
+}

trunk/src/wp-includes/compat.php

@@ -229 +229 @@
  * Internal compat function to mimic mb_strlen().
  *
- * Only understands UTF-8 and 8bit. All other character sets will be treated as 8bit.
- * For `$encoding === UTF-8`, the `$str` input is expected to be a valid UTF-8 byte
- * sequence. The behavior of this function for invalid inputs is undefined.
+ * Only supports UTF-8 and non-shifting single-byte encodings. For all other
+ * encodings expect the counts to be wrong. When the given encoding (or the
+ * `blog_charset` if none is provided) isn’t UTF-8 then the function returns
+ * the byte-count of the provided string.
  *
  * @ignore
@@ -237 +238 @@
  *
  * @param string      $str      The string to retrieve the character length from.
- * @param string|null $encoding Optional. Character encoding to use. Default null.
- * @return int String length of `$str`.
+ * @param string|null $encoding Optional. Count characters according to this encoding.
+ *                              Default is to consult `blog_charset`.
+ * @return int Count of code points if UTF-8, byte length otherwise.
  */
 function _mb_strlen( $str, $encoding = null ) {
-    if ( null === $encoding ) {
-        $encoding = get_option( 'blog_charset' );
-    }
-
-    /*
-     * The solution below works only for UTF-8, so in case of a different charset
-     * just use built-in strlen().
-     */
-    if ( ! _is_utf8_charset( $encoding ) ) {
-        return strlen( $str );
-    }
-
-    if ( _wp_can_use_pcre_u() ) {
-        // Use the regex unicode support to separate the UTF-8 characters into an array.
-        preg_match_all( '/./us', $str, $match );
-        return count( $match[0] );
-    }
-
-    $regex = '/(?:
-        [\x00-\x7F]                  # single-byte sequences   0xxxxxxx
-        | [\xC2-\xDF][\x80-\xBF]       # double-byte sequences   110xxxxx 10xxxxxx
-        | \xE0[\xA0-\xBF][\x80-\xBF]   # triple-byte sequences   1110xxxx 10xxxxxx * 2
-        | [\xE1-\xEC][\x80-\xBF]{2}
-        | \xED[\x80-\x9F][\x80-\xBF]
-        | [\xEE-\xEF][\x80-\xBF]{2}
-        | \xF0[\x90-\xBF][\x80-\xBF]{2} # four-byte sequences   11110xxx 10xxxxxx * 3
-        | [\xF1-\xF3][\x80-\xBF]{3}
-        | \xF4[\x80-\x8F][\x80-\xBF]{2}
-    )/x';
-
-    // Start at 1 instead of 0 since the first thing we do is decrement.
-    $count = 1;
-
-    do {
-        // We had some string left over from the last round, but we counted it in that last round.
-        --$count;
-
-        /*
-         * Split by UTF-8 character, limit to 1000 characters (last array element will contain
-         * the rest of the string).
-         */
-        $pieces = preg_split( $regex, $str, 1000 );
-
-        // Increment.
-        $count += count( $pieces );
-
-        // If there's anything left over, repeat the loop.
-    } while ( $str = array_pop( $pieces ) );
-
-    // Fencepost: preg_split() always returns one extra item in the array.
-    return --$count;
+    return _is_utf8_charset( $encoding ?? get_option( 'blog_charset' ) )
+        ? _wp_utf8_codepoint_count( $str )
+        : strlen( $str );
 }
 

trunk/tests/phpunit/tests/compat/mbStrlen.php

@@ -11 +11 @@
 
     /**
-     * Test that mb_strlen() is always available (either from PHP or WP).
+     * Test that the native mb_strlen() is available.
      */
     public function test_mb_strlen_availability() {
-        $this->assertTrue( function_exists( 'mb_strlen' ) );
+        $this->assertTrue(
+            in_array( 'mb_strlen', get_defined_functions()['internal'], true ),
+            'Test runner should have `mbstring` extension active but doesn’t.'
+        );
     }
 
     /**
-     * @dataProvider data_utf8_string_lengths
+     * @dataProvider data_utf8_strings
      */
-    public function test_mb_strlen( $input_string, $expected_character_length ) {
-        $this->assertSame( $expected_character_length, _mb_strlen( $input_string, 'UTF-8' ) );
+    public function test_mb_strlen( $input_string ) {
+        $this->assertSame(
+            mb_strlen( $input_string, 'UTF-8' ),
+            _mb_strlen( $input_string, 'UTF-8' )
+        );
     }
 
     /**
-     * @dataProvider data_utf8_string_lengths
+     * @dataProvider data_utf8_strings
      */
-    public function test_mb_strlen_via_regex( $input_string, $expected_character_length ) {
-        _wp_can_use_pcre_u( false );
-        $this->assertSame( $expected_character_length, _mb_strlen( $input_string, 'UTF-8' ) );
-        _wp_can_use_pcre_u( 'reset' );
+    public function test_mb_strlen_via_regex( $input_string ) {
+        $this->assertSame(
+            mb_strlen( $input_string, 'UTF-8' ),
+            _mb_strlen( $input_string, 'UTF-8' )
+        );
     }
 
     /**
-     * @dataProvider data_utf8_string_lengths
+     * @dataProvider data_utf8_strings
      */
-    public function test_8bit_mb_strlen( $input_string, $expected_character_length, $expected_byte_length ) {
-        $this->assertSame( $expected_byte_length, _mb_strlen( $input_string, '8bit' ) );
+    public function test_8bit_mb_strlen( $input_string ) {
+        $this->assertSame(
+            mb_strlen( $input_string, '8bit' ),
+            _mb_strlen( $input_string, '8bit' )
+        );
     }
 
@@ -45 +55 @@
      * @return array
      */
-    public function data_utf8_string_lengths() {
+    public function data_utf8_strings() {
         return array(
-            array(
-                'input_string'              => 'баба',
-                'expected_character_length' => 4,
-                'expected_byte_length'      => 8,
-            ),
-            array(
-                'input_string'              => 'баб',
-                'expected_character_length' => 3,
-                'expected_byte_length'      => 6,
-            ),
-            array(
-                'input_string'              => 'I am your б',
-                'expected_character_length' => 11,
-                'expected_byte_length'      => 12,
-            ),
-            array(
-                'input_string'              => '1111111111',
-                'expected_character_length' => 10,
-                'expected_byte_length'      => 10,
-            ),
-            array(
-                'input_string'              => '²²²²²²²²²²',
-                'expected_character_length' => 10,
-                'expected_byte_length'      => 20,
-            ),
-            array(
-                'input_string'              => '３３３３３３３３３３',
-                'expected_character_length' => 10,
-                'expected_byte_length'      => 30,
-            ),
-            array(
-                'input_string'              => '𝟜𝟜𝟜𝟜𝟜𝟜𝟜𝟜𝟜𝟜',
-                'expected_character_length' => 10,
-                'expected_byte_length'      => 40,
-            ),
-            array(
-                'input_string'              => '1²３𝟜1²３𝟜1²３𝟜',
-                'expected_character_length' => 12,
-                'expected_byte_length'      => 30,
-            ),
+            array( 'баба' ),
+            array( 'баб' ),
+            array( 'I am your б' ),
+            array( '1111111111' ),
+            array( '²²²²²²²²²²' ),
+            array( '３３３３３３３３３３' ),
+            array( '𝟜𝟜𝟜𝟜𝟜𝟜𝟜𝟜𝟜𝟜' ),
+            array( '1²３𝟜1²３𝟜1²３𝟜' ),
         );
     }