Changeset 53754

Timestamp:

07/21/2022 09:09:56 PM (4 years ago)

Author:

audrasjb

Message:

Formatting: Normalize to Unicode NFC encoding before converting accent characters in remove_accents().

This changeset adds Unicode sequence normalization from NFD to NFC, via the normalizer_normalize() PHP function which is available with the recommended intl PHP extension.

This fixes an issue where NFD characters were not properly sanitized. It also provides a unit test for NFD sequences (alternate Unicode representations of the same characters).

Props NumidWasNotAvailable, targz, nacin, nunomorgadinho, p_enrique, gitlost, SergeyBiryukov, markoheijnen, mikeschroder, ocean90, pento, helen, rodrigosevero, zodiac1978, ironprogrammer, audrasjb, azaozz, laboiteare, nuryko, virgar, dxd5001, onnimonni, johnbillion.
Fixes #24661, #47763, #35951.
See #30130, #52654.

Location:

trunk

Files:

• src/wp-includes/formatting.php (modified) (2 diffs)
• tests/phpunit/tests/formatting/removeAccents.php (modified) (1 diff)

trunk/src/wp-includes/formatting.php

@@ -1585 +1585 @@
  * @since 5.7.0 Added locale support for `de_AT`.
  * @since 6.0.0 Added the `$locale` parameter.
+ * @since 6.1.0 Added Unicode NFC encoding normalization support.
  *
  * @param string $string Text that might have accent characters.
@@ -1598 +1599 @@
 
         if ( seems_utf8( $string ) ) {
+
+                // Unicode sequence normalization from NFD (Normalization Form Decomposed)
+                // to NFC (Normalization Form [Pre]Composed), the encoding used in this function.
+                if ( function_exists( 'normalizer_normalize' ) ) {
+                        if ( ! normalizer_is_normalized( $string, Normalizer::FORM_C ) ) {
+                                $string = normalizer_normalize( $string, Normalizer::FORM_C );
+                        }
+                }
+
                 $chars = array(
                         // Decompositions for Latin-1 Supplement.

trunk/tests/phpunit/tests/formatting/removeAccents.php

@@ -10 +10 @@
         public function test_remove_accents_simple() {
                 $this->assertSame( 'abcdefghijkl', remove_accents( 'abcdefghijkl' ) );
+        }
+
+        /**
+         * @ticket 24661
+         *
+         * Tests Unicode sequence normalization from NFD (Normalization Form Decomposed)
+         * to NFC (Normalization Form [Pre]Composed), the encoding used in `remove_accents()`.
+         *
+         * For more information on Unicode normalization, see
+         * https://unicode.org/faq/normalization.html.
+         *
+         * @requires extension intl
+         */
+        public function test_remove_accents_latin1_supplement_nfd_encoding() {
+                $input  = 'ªºÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ';
+                $output = 'aoAAAAAAAECEEEEIIIIDNOOOOOOUUUUYTHsaaaaaaaeceeeeiiiidnoooooouuuuythy';
+
+                $this->assertSame( $output, remove_accents( $input ), 'remove_accents replaces Latin-1 Supplement with NFD encoding' );
         }