WordPress.org
Get WordPress

Make WordPress Core

Changeset 58281


Ignore:
Timestamp:
06/02/2024 03:14:35 PM (2 years ago)
Author:
dmsnell
Message:

HTML API: Add custom text decoder.

Provides a custom decoder for strings coming from HTML attributes and
markup. This custom decoder is necessary because of deficiencies in
PHP's html_entity_decode() function:

  • It isn't aware of 720 of the possible named character references in HTML, leaving many out that should be translated.
  • It isn't aware of the ambiguous ampersand rule, which allows conversion of character references in certain contexts when they are missing their closing ;.
  • It doesn't draw a distinction for the ambiguous ampersand rule when decoding attribute values instead of markup values.
  • Use of html_entity_decode() requires manually passing non-default paramter values to ensure it decodes properly.

This decoder also provides some conveniences, such as making a
single-pass and interruptable decode operation possible. This will
provide a number of opportunities to optimize detection and decoding
of things like value prefixes, and whether a value contains a given
substring.

Developed in ​https://github.com/WordPress/wordpress-develop/pull/6387
Discussed in https://core.trac.wordpress.org/ticket/61072

Props dmsnell, gziolo, jonsurrell, jorbin, westonruter, zieladam.
Fixes #61072.

Location:
trunk
Files:
2 added
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/wp-includes/class-wp-token-map.php

    r58188 r58281  
    436436         * @since 6.6.0
    437437         *
    438          * @param string  $word             Determine if this word is a lookup key in the map.
    439          * @param ?string $case_sensitivity 'ascii-case-insensitive' to ignore ASCII case or default of 'case-sensitive'.
     438         * @param string $word             Determine if this word is a lookup key in the map.
     439         * @param string $case_sensitivity Optional. Pass 'ascii-case-insensitive' to ignore ASCII case when matching. Default 'case-sensitive'.
    440440         * @return bool Whether there's an entry for the given word in the map.
    441441         */
    … …  
    522522         *
    523523         * @param string  $text                       String in which to search for a lookup key.
    524          * @param ?int    $offset                     How many bytes into the string where the lookup key ought to start.
    525          * @param ?int    &$matched_token_byte_length Holds byte-length of found token matched, otherwise not set.
    526          * @param ?string $case_sensitivity           'ascii-case-insensitive' to ignore ASCII case or default of 'case-sensitive'.
    527          * @return string|false Mapped value of lookup key if found, otherwise `false`.
     524         * @param int     $offset                     Optional. How many bytes into the string where the lookup key ought to start. Default 0.
     525         * @param ?int    &$matched_token_byte_length Optional. Holds byte-length of found token matched, otherwise not set. Default null.
     526         * @param string  $case_sensitivity           Optional. Pass 'ascii-case-insensitive' to ignore ASCII case when matching. Default 'case-sensitive'.
     527         * @return string|null Mapped value of lookup key if found, otherwise `null`.
    528528         */
    529529        public function read_token( $text, $offset = 0, &$matched_token_byte_length = null, $case_sensitivity = 'case-sensitive' ) {
    … …  
    540540                                return strlen( $this->small_words ) > 0
    541541                                        ? $this->read_small_token( $text, $offset, $matched_token_byte_length, $case_sensitivity )
    542                                         : false;
     542                                        : null;
    543543                        }
    544544
    … …  
    565565                return strlen( $this->small_words ) > 0
    566566                        ? $this->read_small_token( $text, $offset, $matched_token_byte_length, $case_sensitivity )
    567                         : false;
     567                        : null;
    568568        }
    569569
    … …  
    573573         * @since 6.6.0.
    574574         *
    575          * @param string  $text                       String in which to search for a lookup key.
    576          * @param ?int    $offset                     How many bytes into the string where the lookup key ought to start.
    577          * @param ?int    &$matched_token_byte_length Holds byte-length of found lookup key if matched, otherwise not set.
    578          * @param ?string $case_sensitivity           'ascii-case-insensitive' to ignore ASCII case or default of 'case-sensitive'.
    579          * @return string|false Mapped value of lookup key if found, otherwise `false`.
     575         * @param string $text                       String in which to search for a lookup key.
     576         * @param int    $offset                     Optional. How many bytes into the string where the lookup key ought to start. Default 0.
     577         * @param ?int   &$matched_token_byte_length Optional. Holds byte-length of found lookup key if matched, otherwise not set. Default null.
     578         * @param string $case_sensitivity           Optional. Pass 'ascii-case-insensitive' to ignore ASCII case when matching. Default 'case-sensitive'.
     579         * @return string|null Mapped value of lookup key if found, otherwise `null`.
    580580         */
    581581        private function read_small_token( $text, $offset, &$matched_token_byte_length, $case_sensitivity = 'case-sensitive' ) {
    … …  
    617617                }
    618618
    619                 return false;
     619                return null;
    620620        }
    621621
    … …  
    693693         * @since 6.6.0
    694694         *
    695          * @param ?string $indent Use this string for indentation, or rely on the default horizontal tab character.
     695         * @param string $indent Optional. Use this string for indentation, or rely on the default horizontal tab character. Default "\t".
    696696         * @return string Value which can be pasted into a PHP source file for quick loading of table.
    697697         */

  • trunk/src/wp-includes/html-api/class-wp-html-tag-processor.php

    r58233 r58281  
    1616 *    This would increase the size of the changes for some operations but leave more
    1717 *    natural-looking output HTML.
    18  *  - Properly decode HTML character references in `get_attribute()`. PHP's
    19  *    `html_entity_decode()` is wrong in a couple ways: it doesn't account for the
    20  *    no-ambiguous-ampersand rule, and it improperly handles the way semicolons may
    21  *    or may not terminate a character reference.
    2218 *
    2319 * @package WordPress
    … …  
    25002496                 */
    25012497                $enqueued_value = substr( $enqueued_text, $equals_at + 2, -1 );
    2502                 return html_entity_decode( $enqueued_value );
     2498                return WP_HTML_Decoder::decode_attribute( $enqueued_value );
    25032499        }
    25042500
    … …  
    25732569                $raw_value = substr( $this->html, $attribute->value_starts_at, $attribute->value_length );
    25742570
    2575                 return html_entity_decode( $raw_value );
     2571                return WP_HTML_Decoder::decode_attribute( $raw_value );
    25762572        }
    25772573
    … …  
    28732869                }
    28742870
    2875                 $decoded = html_entity_decode( $text, ENT_QUOTES | ENT_HTML5 | ENT_SUBSTITUTE );
     2871                $decoded = WP_HTML_Decoder::decode_text_node( $text );
    28762872
    28772873                /*

  • trunk/src/wp-settings.php

    r58264 r58281  
    254254require ABSPATH . WPINC . '/html-api/class-wp-html-span.php';
    255255require ABSPATH . WPINC . '/html-api/class-wp-html-text-replacement.php';
     256require ABSPATH . WPINC . '/html-api/class-wp-html-decoder.php';
    256257require ABSPATH . WPINC . '/html-api/class-wp-html-tag-processor.php';
    257258require ABSPATH . WPINC . '/html-api/class-wp-html-unsupported-exception.php';

  • trunk/tests/phpunit/tests/html-api/wpHtmlProcessorHtml5lib.php

    r58072 r58281  
    3232         */
    3333        const SKIP_TESTS = array(
    34                 'adoption01/line0046'        => 'Unimplemented: Reconstruction of active formatting elements.',
    35                 'adoption01/line0159'        => 'Unimplemented: Reconstruction of active formatting elements.',
    36                 'adoption01/line0318'        => 'Unimplemented: Reconstruction of active formatting elements.',
    37                 'entities02/line0100'        => 'Encoded characters without semicolon termination in attribute values are not handled properly',
    38                 'entities02/line0114'        => 'Encoded characters without semicolon termination in attribute values are not handled properly',
    39                 'entities02/line0128'        => 'Encoded characters without semicolon termination in attribute values are not handled properly',
    40                 'entities02/line0142'        => 'Encoded characters without semicolon termination in attribute values are not handled properly',
    41                 'entities02/line0156'        => 'Encoded characters without semicolon termination in attribute values are not handled properly',
    42                 'inbody01/line0001'          => 'Bug.',
    43                 'inbody01/line0014'          => 'Bug.',
    44                 'inbody01/line0029'          => 'Bug.',
    45                 'menuitem-element/line0012'  => 'Bug.',
    46                 'plain-text-unsafe/line0001' => 'HTML entities may be mishandled.',
    47                 'plain-text-unsafe/line0105' => 'Binary.',
    48                 'tests1/line0342'            => "Closing P tag implicitly creates opener, which we don't visit.",
    49                 'tests1/line0720'            => 'Unimplemented: Reconstruction of active formatting elements.',
    50                 'tests1/line0833'            => 'Bug.',
    51                 'tests15/line0001'           => 'Unimplemented: Reconstruction of active formatting elements.',
    52                 'tests15/line0022'           => 'Unimplemented: Reconstruction of active formatting elements.',
    53                 'tests2/line0317'            => 'HTML entities may be mishandled.',
    54                 'tests2/line0408'            => 'HTML entities may be mishandled.',
    55                 'tests2/line0650'            => 'Whitespace only test never enters "in body" parsing mode.',
    56                 'tests20/line0497'           => "Closing P tag implicitly creates opener, which we don't visit.",
    57                 'tests23/line0001'           => 'Unimplemented: Reconstruction of active formatting elements.',
    58                 'tests23/line0041'           => 'Unimplemented: Reconstruction of active formatting elements.',
    59                 'tests23/line0069'           => 'Unimplemented: Reconstruction of active formatting elements.',
    60                 'tests23/line0101'           => 'Unimplemented: Reconstruction of active formatting elements.',
    61                 'tests25/line0169'           => 'Bug.',
    62                 'tests26/line0263'           => 'Bug: An active formatting element should be created for a trailing text node.',
    63                 'tests7/line0354'            => 'Bug.',
    64                 'tests8/line0001'            => 'Bug.',
    65                 'tests8/line0020'            => 'Bug.',
    66                 'tests8/line0037'            => 'Bug.',
    67                 'tests8/line0052'            => 'Bug.',
    68                 'webkit01/line0174'          => 'Bug.',
     34                'adoption01/line0046'       => 'Unimplemented: Reconstruction of active formatting elements.',
     35                'adoption01/line0159'       => 'Unimplemented: Reconstruction of active formatting elements.',
     36                'adoption01/line0318'       => 'Unimplemented: Reconstruction of active formatting elements.',
     37                'inbody01/line0001'         => 'Bug.',
     38                'inbody01/line0014'         => 'Bug.',
     39                'inbody01/line0029'         => 'Bug.',
     40                'menuitem-element/line0012' => 'Bug.',
     41                'tests1/line0342'           => "Closing P tag implicitly creates opener, which we don't visit.",
     42                'tests1/line0720'           => 'Unimplemented: Reconstruction of active formatting elements.',
     43                'tests1/line0833'           => 'Bug.',
     44                'tests15/line0001'          => 'Unimplemented: Reconstruction of active formatting elements.',
     45                'tests15/line0022'          => 'Unimplemented: Reconstruction of active formatting elements.',
     46                'tests2/line0650'           => 'Whitespace only test never enters "in body" parsing mode.',
     47                'tests20/line0497'          => "Closing P tag implicitly creates opener, which we don't visit.",
     48                'tests23/line0001'          => 'Unimplemented: Reconstruction of active formatting elements.',
     49                'tests23/line0041'          => 'Unimplemented: Reconstruction of active formatting elements.',
     50                'tests23/line0069'          => 'Unimplemented: Reconstruction of active formatting elements.',
     51                'tests23/line0101'          => 'Unimplemented: Reconstruction of active formatting elements.',
     52                'tests25/line0169'          => 'Bug.',
     53                'tests26/line0263'          => 'Bug: An active formatting element should be created for a trailing text node.',
     54                'tests7/line0354'           => 'Bug.',
     55                'tests8/line0001'           => 'Bug.',
     56                'tests8/line0020'           => 'Bug.',
     57                'tests8/line0037'           => 'Bug.',
     58                'tests8/line0052'           => 'Bug.',
     59                'webkit01/line0174'         => 'Bug.',
    6960        );
    7061
    … …  
    10596                while ( false !== ( $entry = readdir( $handle ) ) ) {
    10697                        if ( ! stripos( $entry, '.dat' ) ) {
    107                                 continue;
    108                         }
    109 
    110                         if ( 'entities01.dat' === $entry || 'entities02.dat' === $entry ) {
    11198                                continue;
    11299                        }

  • trunk/tests/phpunit/tests/wp-token-map/wpTokenMap.php

    r58188 r58281  
    318318
    319319                $skip_bytes = 0;
    320                 $this->assertFalse(
     320                $this->assertNull(
    321321                        $map->read_token( $document, 0, $skip_bytes ),
    322322                        "Shouldn't have found token at start of document."
Note: See TracChangeset for help on using the changeset viewer.