Index: src/wp-includes/compat.php =================================================================== --- src/wp-includes/compat.php (revision 38708) +++ src/wp-includes/compat.php (working copy) @@ -41,6 +41,34 @@ return $utf8_pcre; } +/** + * Returns whether PCRE Unicode character properties (UCP, eg \p{L}) are available for use. + * + * @ignore + * @since 4.7.0 + * @access private + * + * @staticvar string $ucp_pcre + * + * @param bool $set - Used for testing only + * null : default - get PCRE UCP capability + * false : Used for testing - return false for future calls to this function + * 'reset': Used for testing - restore default behavior of this function + */ +function _wp_can_use_pcre_ucp( $set = null ) { + static $ucp_pcre = 'reset'; + + if ( null !== $set ) { + $ucp_pcre = $set; + } + + if ( 'reset' === $ucp_pcre ) { + $ucp_pcre = false !== @preg_match( '/\p{L}/u', '' ); + } + + return $ucp_pcre; +} + if ( ! function_exists( 'mb_substr' ) ) : /** * Compat function to mimic mb_substr(). Index: src/wp-includes/formatting.php =================================================================== --- src/wp-includes/formatting.php (revision 38708) +++ src/wp-includes/formatting.php (working copy) @@ -1711,6 +1711,18 @@ } $string = strtr($string, $chars); + + // Removes "Mn" nonspacing combining marks that follow Latin characters. + if ( _wp_can_use_pcre_ucp() ) { // If UCP available. + $string = preg_replace( '/(?<=\p{Latin})\p{Mn}+/u', '', $string ); + } else { + static $mn_regex_alts, $latin_regex_alts; + if ( ! isset( $mn_regex_alts ) ) { + require dirname( __FILE__ ) . '/unicode/regex_alts.php'; + } + $string = preg_replace( '/(?<=' . $latin_regex_alts . ')(?:' . $mn_regex_alts . ')+/', '', $string ); + } + } else { $chars = array(); // Assume ISO-8859-1 if not UTF-8 Index: src/wp-includes/unicode/regex_alts.php =================================================================== --- src/wp-includes/unicode/regex_alts.php (nonexistent) +++ src/wp-includes/unicode/regex_alts.php (working copy) @@ -0,0 +1,18 @@ +assertEquals( 'Dd', remove_accents( 'Đđ' ) ); } + + /** + * @ticket 24661 + */ + public function test_remove_accents_mn_combining_marks() { + // Creates a string with all Mn nonspacing combining marks (generated from Unicode 5.0.0 "UnicodeData.txt", 880 code points). + // (Using Unicode 5.0.0 to be compatible with PHP 5.2.4 (PCRE 7.2).) + $code_points = array_merge( + range( 0x300, 0x36f ) + , range( 0x483, 0x486 ) + , range( 0x591, 0x5bd ) + , array( 0x5bf ) + , range( 0x5c1, 0x5c2 ) + , range( 0x5c4, 0x5c5 ) + , array( 0x5c7 ) + , range( 0x610, 0x615 ) + , range( 0x64b, 0x65e ) + , array( 0x670 ) + , range( 0x6d6, 0x6dc ) + , range( 0x6df, 0x6e4 ) + , range( 0x6e7, 0x6e8 ) + , range( 0x6ea, 0x6ed ) + , array( 0x711 ) + , range( 0x730, 0x74a ) + , range( 0x7a6, 0x7b0 ) + , range( 0x7eb, 0x7f3 ) + , range( 0x901, 0x902 ) + , array( 0x93c ) + , range( 0x941, 0x948 ) + , array( 0x94d ) + , range( 0x951, 0x954 ) + , range( 0x962, 0x963 ) + , array( 0x981 ) + , array( 0x9bc ) + , range( 0x9c1, 0x9c4 ) + , array( 0x9cd ) + , range( 0x9e2, 0x9e3 ) + , range( 0xa01, 0xa02 ) + , array( 0xa3c ) + , range( 0xa41, 0xa42 ) + , range( 0xa47, 0xa48 ) + , range( 0xa4b, 0xa4d ) + , range( 0xa70, 0xa71 ) + , range( 0xa81, 0xa82 ) + , array( 0xabc ) + , range( 0xac1, 0xac5 ) + , range( 0xac7, 0xac8 ) + , array( 0xacd ) + , range( 0xae2, 0xae3 ) + , array( 0xb01 ) + , array( 0xb3c ) + , array( 0xb3f ) + , range( 0xb41, 0xb43 ) + , array( 0xb4d ) + , array( 0xb56 ) + , array( 0xb82 ) + , array( 0xbc0 ) + , array( 0xbcd ) + , range( 0xc3e, 0xc40 ) + , range( 0xc46, 0xc48 ) + , range( 0xc4a, 0xc4d ) + , range( 0xc55, 0xc56 ) + , array( 0xcbc ) + , array( 0xcbf ) + , array( 0xcc6 ) + , range( 0xccc, 0xccd ) + , range( 0xce2, 0xce3 ) + , range( 0xd41, 0xd43 ) + , array( 0xd4d ) + , array( 0xdca ) + , range( 0xdd2, 0xdd4 ) + , array( 0xdd6 ) + , array( 0xe31 ) + , range( 0xe34, 0xe3a ) + , range( 0xe47, 0xe4e ) + , array( 0xeb1 ) + , range( 0xeb4, 0xeb9 ) + , range( 0xebb, 0xebc ) + , range( 0xec8, 0xecd ) + , range( 0xf18, 0xf19 ) + , array( 0xf35 ) + , array( 0xf37 ) + , array( 0xf39 ) + , range( 0xf71, 0xf7e ) + , range( 0xf80, 0xf84 ) + , range( 0xf86, 0xf87 ) + , range( 0xf90, 0xf97 ) + , range( 0xf99, 0xfbc ) + , array( 0xfc6 ) + , range( 0x102d, 0x1030 ) + , array( 0x1032 ) + , range( 0x1036, 0x1037 ) + , array( 0x1039 ) + , range( 0x1058, 0x1059 ) + , array( 0x135f ) + , range( 0x1712, 0x1714 ) + , range( 0x1732, 0x1734 ) + , range( 0x1752, 0x1753 ) + , range( 0x1772, 0x1773 ) + , range( 0x17b7, 0x17bd ) + , array( 0x17c6 ) + , range( 0x17c9, 0x17d3 ) + , array( 0x17dd ) + , range( 0x180b, 0x180d ) + , array( 0x18a9 ) + , range( 0x1920, 0x1922 ) + , range( 0x1927, 0x1928 ) + , array( 0x1932 ) + , range( 0x1939, 0x193b ) + , range( 0x1a17, 0x1a18 ) + , range( 0x1b00, 0x1b03 ) + , array( 0x1b34 ) + , range( 0x1b36, 0x1b3a ) + , array( 0x1b3c ) + , array( 0x1b42 ) + , range( 0x1b6b, 0x1b73 ) + , range( 0x1dc0, 0x1dca ) + , range( 0x1dfe, 0x1dff ) + , range( 0x20d0, 0x20dc ) + , array( 0x20e1 ) + , range( 0x20e5, 0x20ef ) + , range( 0x302a, 0x302d ) // NOTE: U+302E & U+302F recategorized as Mc in Unicode 6.1.0 + , range( 0x3099, 0x309a ) + , array( 0xa806 ) + , array( 0xa80b ) + , range( 0xa825, 0xa826 ) + , array( 0xfb1e ) + , range( 0xfe00, 0xfe0f ) + , range( 0xfe20, 0xfe23 ) + , range( 0x10a01, 0x10a03 ) + , range( 0x10a05, 0x10a06 ) + , range( 0x10a0c, 0x10a0f ) + , range( 0x10a38, 0x10a3a ) + , array( 0x10a3f ) + , range( 0x1d167, 0x1d169 ) + , range( 0x1d17b, 0x1d182 ) + , range( 0x1d185, 0x1d18b ) + , range( 0x1d1aa, 0x1d1ad ) + , range( 0x1d242, 0x1d244 ) + , range( 0xe0100, 0xe01ef ) + ); + $combining_marks = 'a'; // Only if follow Latin character. + foreach ( $code_points as $c ) { + if ( $c < 0x800 ) { + $combining_marks .= chr( 0xc0 | $c >> 6 ) . chr( 0x80 | $c & 0x3f ); + } elseif ( $c < 0x10000 ) { + $combining_marks .= chr( 0xe0 | $c >> 12 ) . chr( 0x80 | $c >> 6 & 0x3f ) . chr( 0x80 | $c & 0x3f ); + } else { + $combining_marks .= chr( 0xf0 | $c >> 18 ) . chr( 0x80 | $c >> 12 & 0x3f ) . chr( 0x80 | $c >> 6 & 0x3f ) . chr( 0x80 | $c & 0x3f ); + } + } + // Performs the test: all the characters should be removed. + $this->assertSame( 'a', remove_accents( $combining_marks ) ); + if ( _wp_can_use_pcre_ucp() ) { // If UCP available. + // Test single-byte replace also. + _wp_can_use_pcre_ucp( false ); + $this->assertSame( 'a', remove_accents( $combining_marks ) ); + _wp_can_use_pcre_ucp( 'reset' ); + } + } + + /** + * @ticket 24661 + * @dataProvider data_remove_accents_combining + */ + function test_remove_accents_combining( $input, $output ) { + $this->assertSame( $output, remove_accents( $input ) ); + } + + function data_remove_accents_combining() { + return array( + // Test a collection of filenames that could have problems. + array( + "Capture d’e\xcc\x81cran 2013-02-20 a\xcc\x80 23.36.06.png", // NOTE: apostrophe is U+2019 + //"Capture d'ecran 2013-02-20 a 23.36.06.png", // NOTE: apostrophe should go to U+0029 + "Capture d’ecran 2013-02-20 a 23.36.06.png", + ), + array( + "Buttermo\xcc\x88deli.jpg", + "Buttermodeli.jpg", + ), + array( + "Mu\xcc\x88nsterl.Mai13a.jpg", + "Munsterl.Mai13a.jpg", + ), + + array( + 'Cáo nâu lanh lẹ nhảy qua người lười biếng. Do bạch kim rất quý, sẽ để lắp vô xương', + "Cao nau lanh le nhay qua nguoi luoi bieng. Do bach kim rat quy, se de lap vo xuong", + ), + array( + 'Příliš žluťoučký kůň úpěl ďábelské kódy. Pójdźże, kiń tę chmurność w głąb flaszy! Päťtýždňové vĺčatá nervózne štekajú na môjho ďatľa v tŕní.', + "Prilis zlutoucky kun upel dabelske kody. Pojdzze, kin te chmurnosc w glab flaszy! Pattyzdnove vlcata nervozne stekaju na mojho datla v trni.", + ), + array( + "Les naïfs ægithales hâtifs pondant à noël où il gèle sont sûrs d'être déçus en voyant leurs drôles d'œufs abîmés.", + "Les naifs aegithales hatifs pondant a noel ou il gele sont surs d'etre decus en voyant leurs droles d'oeufs abimes.", + ), + array( + 'a᷄ (a + combining macron-acute). +⃟ (plus with enclosing diamond). a︠e︡ (ae with a combining ligature)', + "a (a + combining macron-acute). +⃟ (plus with enclosing diamond). ae (ae with a combining ligature)", // NOTE: Not doing "Me" enclosing combining marks. + ), + + // From UTF-8 Sampler http://www.columbia.edu/~fdc/utf8/ + array( + // NOTE: U+200C ZERO WIDTH NON-JOINER not dealt with. + "Im finſteren Jagdſchloß am offenen Felsquellwaſſer patzte der affig-flatterhafte kauzig-höf\xe2\x80\x8cliche Bäcker über ſeinem verſifften kniffligen C-Xylophon.", + "Im finsteren Jagdschlos am offenen Felsquellwasser patzte der affig-flatterhafte kauzig-hof\xe2\x80\x8cliche Backer uber seinem versifften kniffligen C-Xylophon.", + ), + ); + } }