Index: src/wp-includes/formatting.php =================================================================== --- src/wp-includes/formatting.php (revision 38522) +++ src/wp-includes/formatting.php (working copy) @@ -1700,6 +1700,17 @@ } $string = strtr($string, $chars); + + // Removes "Mn" nonspacing combining marks that follow Latin characters. + if ( false !== @preg_match( '/\p{L}/u', '' ) ) { // If UCP available. + $string = preg_replace( '/(?<=\p{Latin})\p{Mn}+/u', '', $string ); + } else { + if ( ! defined( 'WP_MN_REGEX_ALTS' ) ) { + require dirname( __FILE__ ) . '/unicode/regex_alts.php'; + } + $string = preg_replace( '/(?<=' . WP_LATIN_REGEX_ALTS . ')(?:' . WP_MN_REGEX_ALTS . ')+/', '', $string ); + } + } else { $chars = array(); // Assume ISO-8859-1 if not UTF-8 Index: src/wp-includes/unicode/regex_alts.php =================================================================== --- src/wp-includes/unicode/regex_alts.php (nonexistent) +++ src/wp-includes/unicode/regex_alts.php (working copy) @@ -0,0 +1,18 @@ +assertEquals( 'al·lallalla', remove_accents( 'al·lallaŀla' ) ); } + + /** + * @ticket 24661 + */ + public function test_remove_accents_mn_combining_marks() { + // Creates a string with some Mn nonspacing combining marks. + $code_points = array_merge( + range( 0x0300, 0x036F ) // Combining Diacritical Marks + , range( 0x1DC0, 0x1DF5 ) // Combining Diacritical Marks Supplement + , range( 0x20D0, 0x20DC ) // Combining Diacritical Marks for Symbols + , range( 0x20E5, 0x20F0 ) // Combining Diacritical Marks for Symbols + , range( 0xFE20, 0xFE2D ) // Combining Half Marks (restricted to Unicode 7.0) + ); + $combining_marks = 'a'; // Only if follow Latin character. + foreach ( $code_points as $code_point ) { + if ( $code_point <= 0x07ff ) { + $combining_marks .= chr( 0xc0 | ( $code_point >> 6 ) ) . chr( 0x80 | ( $code_point & 0x003f ) ); + } else { + $combining_marks .= chr( 0xe0 | ( $code_point >> 12 ) ) . chr( 0x80 | ( ( $code_point >> 6 ) & 0x003f ) ) . chr( 0x80 | ( $code_point & 0x003f ) ); + } + } + // Performs the test: all the characters should be removed. + $this->assertSame( 'a', remove_accents( $combining_marks ) ); + + if ( false !== @preg_match( '/\p{L}/u', '' ) ) { // If UCP available. + // Test single-byte replace also. + $string = $combining_marks; + if ( ! defined( 'WP_MN_REGEX_ALTS' ) ) { + require ABSPATH . WPINC . '/unicode/regex_alts.php'; + } + $string = preg_replace( '/(?<=' . WP_LATIN_REGEX_ALTS . ')(?:' . WP_MN_REGEX_ALTS . ')+/', '', $string ); + $this->assertSame( 'a', $string ); + } + } + + /** + * @ticket 24661 + * @dataProvider data_remove_accents_combining + */ + function test_remove_accents_combining( $input, $output ) { + $this->assertSame( $output, remove_accents( $input ) ); + } + + function data_remove_accents_combining() { + return array( + // Test a collection of filenames that could have problems. + array( + "Capture d’e\xcc\x81cran 2013-02-20 a\xcc\x80 23.36.06.png", // NOTE: apostrophe is U+2019 + //"Capture d'ecran 2013-02-20 a 23.36.06.png", // NOTE: apostrophe should go to U+0029 + "Capture d’ecran 2013-02-20 a 23.36.06.png", + ), + array( + "Buttermo\xcc\x88deli.jpg", + "Buttermodeli.jpg", + ), + array( + "Mu\xcc\x88nsterl.Mai13a.jpg", + "Munsterl.Mai13a.jpg", + ), + + array( + 'Cáo nâu lanh lẹ nhảy qua người lười biếng. Do bạch kim rất quý, sẽ để lắp vô xương', + "Cao nau lanh le nhay qua nguoi luoi bieng. Do bach kim rat quy, se de lap vo xuong", + ), + array( + 'Příliš žluťoučký kůň úpěl ďábelské kódy. Pójdźże, kiń tę chmurność w głąb flaszy! Päťtýždňové vĺčatá nervózne štekajú na môjho ďatľa v tŕní.', + "Prilis zlutoucky kun upel dabelske kody. Pojdzze, kin te chmurnosc w glab flaszy! Pattyzdnove vlcata nervozne stekaju na mojho datla v trni.", + ), + array( + "Les naïfs ægithales hâtifs pondant à noël où il gèle sont sûrs d'être déçus en voyant leurs drôles d'œufs abîmés.", + "Les naifs aegithales hatifs pondant a noel ou il gele sont surs d'etre decus en voyant leurs droles d'oeufs abimes.", + ), + array( + 'a᷄ (a + combining macron-acute). +⃟ (plus with enclosing diamond). a︠e︡ (ae with a combining ligature)', + "a (a + combining macron-acute). +⃟ (plus with enclosing diamond). ae (ae with a combining ligature)", // NOTE: Not doing "Me" enclosing combining marks. + ), + + // From UTF-8 Sampler http://www.columbia.edu/~fdc/utf8/ + array( + // NOTE: U+200C ZERO WIDTH NON-JOINER not dealt with. + "Im finſteren Jagdſchloß am offenen Felsquellwaſſer patzte der affig-flatterhafte kauzig-höf\xe2\x80\x8cliche Bäcker über ſeinem verſifften kniffligen C-Xylophon.", + "Im finsteren Jagdschlos am offenen Felsquellwasser patzte der affig-flatterhafte kauzig-hof\xe2\x80\x8cliche Backer uber seinem versifften kniffligen C-Xylophon.", + ), + ); + } }