Ticket #29717: bug29717.k1.patch

wp-includes/formatting.php

@@ -1105 +1105 @@
  *
  * @param string  $string The text which is to be checked.
  * @param bool    $strip Optional. Whether to attempt to strip out invalid UTF8. Default is false.
- * @return string The checked text.
+ * @param string  $bytewise Optional. Choose '' to use bytewise regex only for implementing $strip,
+                  'fallback' to use bytewise regex when UTF8 regex is unavailable,
+ *                'always' to use bytewise regex always. Default is ''.
+ * @return string Unmodifed string if valid UTF8; or blog_charset is not UTF8; or UTF8 regex unavailable and $bytewise falsey.
+ *                Otherwise, empty string or string stripped of invalid chars.
  */
-function wp_check_invalid_utf8( $string, $strip = false ) {
+function wp_check_invalid_utf8( $string, $strip = false, $bytewise = '' ) {
         $string = (string) $string;
 
-        if ( 0 === strlen( $string ) ) {
+        if ( '' == $string ) {
                 return '';
         }
 
@@ -1126 +1130 @@
         // Check for support for utf8 in the installed PCRE library once and store the result in a static
         static $utf8_pcre = null;
         if ( ! isset( $utf8_pcre ) ) {
-                $utf8_pcre = @preg_match( '/^./u', 'a' );
+                if ( 'always' === $bytewise ) {
+                        $utf8_pcre = false;
+                } else {
+                        $utf8_pcre = @preg_match( '/^./u', 'a' );  // Returns false on error.
+                }
         }
-        // We can't demand utf8 in the PCRE installation, so just return the string in those cases
-        if ( ! $utf8_pcre ) {
+        if ( $utf8_pcre ) {
+                // preg_match fails when it encounters invalid UTF8 in $string
+                if ( 1 === @preg_match( '/^./us', $string ) ) {
+                        return $string;
+                }
+                if ( ! $strip ) {
+                        return '';
+                }
+        } elseif ( ! $bytewise ) {
                 return $string;
         }
+        if ( $strip || $bytewise ) {  // $strip, or $bytewise is 'fallback' or 'always'
+                // Bytewise regex captures valid UTF8 in blocks of 40 and skips invalid characters.
+                $regex = '/
+                        (
+                                (?: [\x00-\x7F]                   # single-byte sequences   0xxxxxxx
+                                |   [\xC2-\xDF][\x80-\xBF]        # double-byte sequences   110xxxxx 10xxxxxx
+                                |   \xE0[\xA0-\xBF][\x80-\xBF]    # triple-byte sequences   1110xxxx 10xxxxxx * 2
+                                |   [\xE1-\xEC][\x80-\xBF]{2}
+                                |   \xED[\x80-\x9F][\x80-\xBF]
+                                |   [\xEE-\xEF][\x80-\xBF]{2}
+                                |   \xF0[\x90-\xBF][\x80-\xBF]{2} # four-byte sequences   11110xxx 10xxxxxx * 3
+                                |   [\xF1-\xF3][\x80-\xBF]{3}
+                                |   \xF4[\x80-\x8F][\x80-\xBF]{2}
+                                ){1,40}                           # ...one or more times
+                        ) | .                                 # anything else
+                        /x';
 
-        // preg_match fails when it encounters invalid UTF8 in $string
-        if ( 1 === @preg_match( '/^./us', $string ) ) {
-                return $string;
+                // Remove invalid byte sequences.
+                $clean_string = @preg_replace( $regex, '$1', $string );  // Returns null on error.
+                if ( $clean_string === $string ) {
+                        return $string;
+                }
+                // Return string with invalid characters stripped if requested (not recommended).
+                if ( $strip && is_string( $clean_string ) ) {
+                        return $clean_string;
+                }
         }
-
-        // Attempt to strip the bad chars if requested (not recommended)
-        if ( $strip && function_exists( 'iconv' ) ) {
-                return iconv( 'utf-8', 'utf-8', $string );
-        }
-
         return '';
 }