Ticket #22692: miqro-22692.12.patch

src/wp-includes/formatting.php

@@ -73 +73 @@
                 $static_characters = array_merge( array( '---', ' -- ', '--', ' - ', 'xn–', '...', '``', '\'\'', ' (tm)' ), $cockney );
                 $static_replacements = array_merge( array( $em_dash, ' ' . $em_dash . ' ', $en_dash, ' ' . $en_dash . ' ', 'xn--', '…', $opening_quote, $closing_quote, ' ™' ), $cockneyreplace );
 
+                // Pattern-based replacements of characters.
                 $dynamic = array();
-                if ( "'" != $apos ) {
-                        $dynamic[ '/\'(\d\d(?:’|\')?s)/' ] = $apos . '$1'; // '99's
-                        $dynamic[ '/\'(\d)/'                   ] = $apos . '$1'; // '99
-                }
+                $spaces = wp_spaces_regexp();
+
+                // '99s (apostrophe)
+                if ( "'" != $apos )
+                        $dynamic[ '/\'(?=\d)/' ] = $apos;
+
+                // Single quote at start, or preceded by (, {, <, [, ", or spaces.
                 if ( "'" != $opening_single_quote )
-                        $dynamic[ '/(\s|\A|[([{<]|")\'/'       ] = '$1' . $opening_single_quote; // opening single quote, even after (, {, <, [
+                        $dynamic[ '/(?<=\A|[([{<"]|' . $spaces . ')\'/' ] = $opening_single_quote;
+
+                // 9" (double prime)
                 if ( '"' != $double_prime )
-                        $dynamic[ '/(\d)"/'                    ] = '$1' . $double_prime; // 9" (double prime)
+                        $dynamic[ '/(?<=\d)"/' ] = $double_prime;
+
+                // 9' (prime)
                 if ( "'" != $prime )
-                        $dynamic[ '/(\d)\'/'                   ] = '$1' . $prime; // 9' (prime)
+                        $dynamic[ '/(?<=\d)\'/' ] = $prime;
+
+                // Apostrophe in a word.  No spaces or double primes.
                 if ( "'" != $apos )
-                        $dynamic[ '/(\S)\'([^\'\s])/'          ] = '$1' . $apos . '$2'; // apostrophe in a word
+                        $dynamic[ '/(?<!' . $spaces . ')\'(?!\'|' . $spaces . ')/' ] = $apos;
+
+                // Double quote at start, or preceded by (, {, <, [, or spaces, and not followed by spaces.
                 if ( '"' != $opening_quote )
-                        $dynamic[ '/(\s|\A|[([{<])"(?!\s)/'    ] = '$1' . $opening_quote . '$2'; // opening double quote, even after (, {, <, [
+                        $dynamic[ '/(?<=\A|[([{<]|' . $spaces . ')"(?!' . $spaces . ')/' ] = $opening_quote;
+
+                // Any remaining double quotes.
                 if ( '"' != $closing_quote )
-                        $dynamic[ '/"(\s|\S|\Z)/'              ] = $closing_quote . '$1'; // closing double quote
+                        $dynamic[ '/"/' ] = $closing_quote;
+
+                // Single quotes followed by spaces or a period.
                 if ( "'" != $closing_single_quote )
-                        $dynamic[ '/\'([\s.]|\Z)/'             ] = $closing_single_quote . '$1'; // closing single quote
+                        $dynamic[ '/\'(?=\Z|\.|' . $spaces . ')/' ] = $closing_single_quote; 
 
-                $dynamic[ '/\b(\d+)x(\d+)\b/'              ] = '$1&#215;$2'; // 9x9 (times)
+                // 9x9 (times)
+                $dynamic[ '/\b(\d+)x(\d+)\b/' ] = '$1&#215;$2';
 
                 $dynamic_characters = array_keys( $dynamic );
                 $dynamic_replacements = array_values( $dynamic );
@@ -318 +335 @@
         }
 
         $tagregexp = join( '|', array_map( 'preg_quote', array_keys( $shortcode_tags ) ) );
+        $spaces = wp_spaces_regexp();
 
         $pattern =
                   '/'
                 . '<p>'                              // Opening paragraph
-                . '\\s*+'                            // Optional leading whitespace
+                . '(?:' . $spaces . ')*+'            // Optional leading whitespace
                 . '('                                // 1: The shortcode
                 .     '\\['                          // Opening bracket
                 .     "($tagregexp)"                 // 2: Shortcode name
@@ -347 +365 @@
                 .         ')?'
                 .     ')'
                 . ')'
-                . '\\s*+'                            // optional trailing whitespace
+                . '(?:' . $spaces . ')*+'            // optional trailing whitespace
                 . '<\\/p>'                           // closing paragraph
                 . '/s';
 
@@ -1865 +1883 @@
          */
         $src_url = apply_filters( 'smilies_src', includes_url( "images/smilies/$img" ), $img, site_url() );
 
-        return sprintf( ' <img src="%s" alt="%s" class="wp-smiley" /> ', esc_url( $src_url ), esc_attr( $smiley ) );
+        return sprintf( '<img src="%s" alt="%s" class="wp-smiley" />', esc_url( $src_url ), esc_attr( $smiley ) );
 }
 
 /**
@@ -3781 +3799 @@
 
         return false;
 }
+
+/**
+ * Returns the regexp for common whitespace characters.
+ *
+ * By default, spaces include new lines, tabs, nbsp entities, and the UTF-8 nbsp.
+ * This is designed to replace the PCRE \s sequence.  In ticket #22692, that
+ * sequence was found to be unreliable due to random inclusion of the A0 byte.
+ *
+ * @since 3.9.0
+ *
+ * @return string The spaces regexp.
+ */
+function wp_spaces_regexp() {
+        static $spaces;
+
+        if (empty($spaces)) {
+                $spaces = apply_filters( 'wp_spaces_regexp', '[\r\n\t ]|\xC2\xA0|&nbsp;' );
+        }
+
+        return $spaces;
+}

src/wp-includes/functions.php

@@ -2560 +2560 @@
          */
         krsort($wpsmiliestrans);
 
-        $wp_smiliessearch = '/((?:\s|^)';
+        $spaces = wp_spaces_regexp();
 
+        // Begin first "subpattern"
+        $wp_smiliessearch = '/(?<=' . $spaces . '|^)';
+
         $subchar = '';
         foreach ( (array) $wpsmiliestrans as $smiley => $img ) {
                 $firstchar = substr($smiley, 0, 1);
@@ -2570 +2573 @@
                 // new subpattern?
                 if ($firstchar != $subchar) {
                         if ($subchar != '') {
-                                $wp_smiliessearch .= ')(?=\s|$))|((?:\s|^)'; ;
+                                $wp_smiliessearch .= ')(?=' . $spaces . '|$)';  // End previous "subpattern"
+                                $wp_smiliessearch .= '|(?<=' . $spaces . '|^)'; // Begin another "subpattern"
                         }
                         $subchar = $firstchar;
                         $wp_smiliessearch .= preg_quote($firstchar, '/') . '(?:';
@@ -2580 +2584 @@
                 $wp_smiliessearch .= preg_quote($rest, '/');
         }
 
-        $wp_smiliessearch .= ')(?=\s|$))/m';
+        $wp_smiliessearch .= ')(?=' . $spaces . '|$)/m';
 
 }
 

tests/phpunit/tests/formatting/Smilies.php

@@ -275 +275 @@
 
                 $wpsmiliestrans = $orig_trans; // reset original translations array
         }
-}
- No newline at end of file
+
+        /**
+         * Check that $wp_smiliessearch pattern will match smilies
+         * between spaces, but never capture those spaces.
+         *
+         * Further check that spaces aren't randomly deleted
+         * or added when replacing the text with an image.
+         *
+         * @ticket 22692
+         */
+        function test_spaces_around_smilies() {
+                $nbsp = "\xC2\xA0";
+
+                $input  = array();
+                $output = array();
+
+                $input[]  = 'My test :) smile';
+                $output[] = array('test <img ', 'alt=":)"', ' /> smile');
+
+                $input[]  = 'My test ;) smile';
+                $output[] = array('test <img ', 'alt=";)"', ' /> smile');
+
+                $input[]  = 'My test &nbsp;:)&nbsp;smile';
+                $output[] = array('test &nbsp;<img ', 'alt=":)"', ' />&nbsp;smile');
+
+                $input[]  = 'My test &nbsp;;)&nbsp;smile';
+                $output[] = array('test &nbsp;<img ', 'alt=";)"', ' />&nbsp;smile');
+
+                $input[]  = "My test {$nbsp}:){$nbsp}smile";
+                $output[] = array("test {$nbsp}<img ", 'alt=":)"', " />{$nbsp}smile");
+
+                $input[]  = "My test {$nbsp};){$nbsp}smile";
+                $output[] = array("test {$nbsp}<img ", 'alt=";)"', " />{$nbsp}smile");
+
+                foreach($input as $key => $in) {
+                        $result = convert_smilies( $in );
+                        foreach($output[$key] as $out) {
+
+                                // Each output element must appear in the results.
+                                $this->assertContains( $out, $result );
+
+                        }
+                }
+        }
+}

tests/phpunit/tests/formatting/WPTexturize.php

@@ -14 +14 @@
                 $this->assertEquals('[a]a–b[code]---[/code]a–b[/a]', wptexturize('[a]a--b[code]---[/code]a--b[/a]'));
                 $this->assertEquals('<pre><code></code>--</pre>', wptexturize('<pre><code></code>--</pre>'));
 
-                $this->assertEquals('<code>---</code>', wptexturize('<code>---</code>'));
+                $this->assertEquals( '<code>---</code>',     wptexturize( '<code>---</code>'     ) );
+                $this->assertEquals( '<kbd>---</kbd>',       wptexturize( '<kbd>---</kbd>'       ) );
+                $this->assertEquals( '<style>---</style>',   wptexturize( '<style>---</style>'   ) );
+                $this->assertEquals( '<script>---</script>', wptexturize( '<script>---</script>' ) );
+                $this->assertEquals( '<tt>---</tt>',         wptexturize( '<tt>---</tt>'         ) );
 
                 $this->assertEquals('<code>href="baba"</code> &#8220;baba&#8221;', wptexturize('<code>href="baba"</code> "baba"'));
 
@@ -44 +48 @@
         //WP Ticket #4539
         function test_basic_quotes() {
                 $this->assertEquals('test&#8217;s', wptexturize('test\'s'));
-                $this->assertEquals('test&#8217;s', wptexturize('test\'s'));
 
                 $this->assertEquals('&#8216;quoted&#8217;', wptexturize('\'quoted\''));
                 $this->assertEquals('&#8220;quoted&#8221;', wptexturize('"quoted"'));
@@ -184 +187 @@
          * @ticket 23185
          */
         function test_spaces_around_hyphens() {
+                $nbsp = "\xC2\xA0";
+
                 $this->assertEquals( ' &#8211; ', wptexturize( ' - ' ) ); 
                 $this->assertEquals( '&nbsp;&#8211;&nbsp;', wptexturize( '&nbsp;-&nbsp;' ) );
                 $this->assertEquals( ' &#8211;&nbsp;', wptexturize( ' -&nbsp;' ) );
                 $this->assertEquals( '&nbsp;&#8211; ', wptexturize( '&nbsp;- ') );
+                $this->assertEquals( "$nbsp&#8211;$nbsp", wptexturize( "$nbsp-$nbsp" ) );
+                $this->assertEquals( " &#8211;$nbsp", wptexturize( " -$nbsp" ) );
+                $this->assertEquals( "$nbsp&#8211; ", wptexturize( "$nbsp- ") );
 
                 $this->assertEquals( ' &#8212; ', wptexturize( ' -- ' ) ); 
                 $this->assertEquals( '&nbsp;&#8212;&nbsp;', wptexturize( '&nbsp;--&nbsp;' ) );
                 $this->assertEquals( ' &#8212;&nbsp;', wptexturize( ' --&nbsp;' ) );
                 $this->assertEquals( '&nbsp;&#8212; ', wptexturize( '&nbsp;-- ') );
+                $this->assertEquals( "$nbsp&#8212;$nbsp", wptexturize( "$nbsp--$nbsp" ) );
+                $this->assertEquals( " &#8212;$nbsp", wptexturize( " --$nbsp" ) );
+                $this->assertEquals( "$nbsp&#8212; ", wptexturize( "$nbsp-- ") );
         }
+
+        /**
+         * @ticket 22692
+         */
+        function test_spaces_around_quotes() {
+                $nbsp = "\xC2\xA0";
+                $pi   = "\xCE\xA0";
+
+                // This should never happen, even if the desired output changes some day.
+
+                $problem_input  = "$nbsp\"A";
+                $problem_output = "$nbsp&#8221;A";
+
+                $this->assertNotEquals( $problem_output, wptexturize( $problem_input ) );
+
+                // These are desirable outputs for the current design.
+
+                $input  = array();
+                $output = array();
+
+                $input[]  = "stop. $nbsp\"A quote after 2 spaces.\"";
+                $output[] = "stop. $nbsp&#8220;A quote after 2 spaces.&#8221;";
+
+                $input[]  = "stop.$nbsp$nbsp\"A quote after 2 spaces.\"";
+                $output[] = "stop.$nbsp$nbsp&#8220;A quote after 2 spaces.&#8221;";
+
+                $input[]  = "stop. $nbsp'A quote after 2 spaces.'";
+                $output[] = "stop. $nbsp&#8216;A quote after 2 spaces.&#8217;";
+
+                $input[]  = "stop.$nbsp$nbsp'A quote after 2 spaces.'";
+                $output[] = "stop.$nbsp$nbsp&#8216;A quote after 2 spaces.&#8217;";
+
+                $input[]  = "stop. &nbsp;\"A quote after 2 spaces.\"";
+                $output[] = "stop. &nbsp;&#8220;A quote after 2 spaces.&#8221;";
+
+                $input[]  = "stop.&nbsp;&nbsp;\"A quote after 2 spaces.\"";
+                $output[] = "stop.&nbsp;&nbsp;&#8220;A quote after 2 spaces.&#8221;";
+
+                $input[]  = "stop. &nbsp;'A quote after 2 spaces.'";
+                $output[] = "stop. &nbsp;&#8216;A quote after 2 spaces.&#8217;";
+
+                $input[]  = "stop.&nbsp;&nbsp;'A quote after 2 spaces.'";
+                $output[] = "stop.&nbsp;&nbsp;&#8216;A quote after 2 spaces.&#8217;";
+
+                $input[]  = "Contraction: $pi's";
+                $output[] = "Contraction: $pi&#8217;s";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_apos_before_digits() {
+                $input  = array();
+                $output = array();
+
+                // Apostrophe before a number always becomes &#8217 (apos);
+
+                $input[]  = "word '99 word";
+                $output[] = "word &#8217;99 word";
+
+                $input[]  = "word'99 word";
+                $output[] = "word&#8217;99 word";
+
+                $input[]  = "word '99word";
+                $output[] = "word &#8217;99word";
+
+                $input[]  = "word'99word";
+                $output[] = "word&#8217;99word";
+
+                $input[]  = "word '99&#8217;s word"; // Appears as a separate but logically superfluous pattern in 3.8.
+                $output[] = "word &#8217;99&#8217;s word";
+
+                $input[]  = "word '99's word"; // Due to the logic error, second apos becomes a prime.  See ticket #22823
+                $output[] = "word &#8217;99&#8242;s word";
+
+                $input[]  = "word '99'samsonite";
+                $output[] = "word &#8217;99&#8242;samsonite";
+
+                $input[]  = "according to our source, '33% of all students scored less than 50' on the test."; // Apostrophes and primes have priority over quotes
+                $output[] = "according to our source, &#8217;33% of all students scored less than 50&#8242; on the test.";
+
+                $input[]  = "word '99' word"; // See ticket #8775
+                $output[] = "word &#8217;99&#8242; word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_opening_single_quote() {
+                $input  = array();
+                $output = array();
+
+                // Apostrophe after a space or ([{<" becomes &#8216; (opening_single_quote)
+
+                $input[]  = "word 'word word";
+                $output[] = "word &#8216;word word";
+
+                $input[]  = "word ('word word";
+                $output[] = "word (&#8216;word word";
+
+                $input[]  = "word ['word word";
+                $output[] = "word [&#8216;word word";
+
+                $input[]  = "word <'word word"; // Invalid HTML input?
+                $output[] = "word <&#8216;word word";
+
+                $input[]  = "word &lt;'word word"; // Valid HTML input triggers the apos in a word pattern
+                $output[] = "word &lt;&#8217;word word";
+
+                $input[]  = "word {'word word";
+                $output[] = "word {&#8216;word word";
+
+                $input[]  = "word \"'word word";
+                $output[] = "word &#8220;&#8216;word word"; // Two opening quotes
+
+                $input[]  = "'word word";
+                $output[] = "&#8216;word word";
+
+                $input[]  = "word('word word";
+                $output[] = "word(&#8216;word word";
+
+                $input[]  = "word['word word";
+                $output[] = "word[&#8216;word word";
+
+                $input[]  = "word<'word word";
+                $output[] = "word<&#8216;word word";
+
+                $input[]  = "word&lt;'word word";
+                $output[] = "word&lt;&#8217;word word";
+
+                $input[]  = "word{'word word";
+                $output[] = "word{&#8216;word word";
+
+                $input[]  = "word\"'word word";
+                $output[] = "word&#8221;&#8216;word word"; // Closing quote, then opening quote
+
+                $input[]  = "word ' word word";
+                $output[] = "word &#8216; word word";
+
+                $input[]  = "word (' word word";
+                $output[] = "word (&#8216; word word";
+
+                $input[]  = "word [' word word";
+                $output[] = "word [&#8216; word word";
+
+                $input[]  = "word <' word word"; // Invalid HTML input?
+                $output[] = "word <&#8216; word word";
+
+                $input[]  = "word &lt;' word word"; // Valid HTML input triggers the closing single quote here
+                $output[] = "word &lt;&#8217; word word";
+
+                $input[]  = "word {' word word";
+                $output[] = "word {&#8216; word word";
+
+                $input[]  = "word \"' word word";
+                $output[] = "word &#8220;&#8216; word word"; // Two opening quotes
+
+                $input[]  = "' word word";
+                $output[] = "&#8216; word word";
+
+                $input[]  = "word(' word word";
+                $output[] = "word(&#8216; word word";
+
+                $input[]  = "word[' word word";
+                $output[] = "word[&#8216; word word";
+
+                $input[]  = "word<' word word";
+                $output[] = "word<&#8216; word word";
+
+                $input[]  = "word&lt;' word word";
+                $output[] = "word&lt;&#8217; word word";
+
+                $input[]  = "word{' word word";
+                $output[] = "word{&#8216; word word";
+
+                $input[]  = "word\"' word word";
+                $output[] = "word&#8221;&#8216; word word"; // Closing quote, then opening quote
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_double_prime() {
+                $input  = array();
+                $output = array();
+
+                // Double quote after a number becomes &#8243; (double_prime)
+
+                $input[]  = 'word 99" word';
+                $output[] = 'word 99&#8243; word';
+
+                $input[]  = 'word 99"word';
+                $output[] = 'word 99&#8243;word';
+
+                $input[]  = 'word99" word';
+                $output[] = 'word99&#8243; word';
+
+                $input[]  = 'word99"word';
+                $output[] = 'word99&#8243;word';
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_single_prime() {
+                $input  = array();
+                $output = array();
+
+                // Apostrophe after a number becomes &#8242; (prime)
+
+                $input[]  = "word 99' word";
+                $output[] = "word 99&#8242; word";
+
+                $input[]  = "word 99'word";
+                $output[] = "word 99&#8242;word";
+
+                $input[]  = "word99' word";
+                $output[] = "word99&#8242; word";
+
+                $input[]  = "word99'word";
+                $output[] = "word99&#8242;word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_contractions() {
+                $input  = array();
+                $output = array();
+
+                // Apostrophe "in a word" becomes &#8217; (apos)
+
+                $input[]  = "word word's word";
+                $output[] = "word word&#8217;s word";
+
+                $input[]  = "word word'. word"; // Quotes with outside punctuation could end with apostrophes instead of closing quotes (may affect i18n)
+                $output[] = "word word&#8217;. word";
+
+                $input[]  = "word ]'. word";
+                $output[] = "word ]&#8217;. word";
+
+                $input[]  = "word )'. word";
+                $output[] = "word )&#8217;. word";
+
+                $input[]  = "word }'. word";
+                $output[] = "word }&#8217;. word";
+
+                $input[]  = "word >'. word"; // Not tested
+                $output[] = "word >&#8217;. word";
+
+                $input[]  = "word &gt;'. word";
+                $output[] = "word &gt;&#8217;. word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_opening_quote() {
+                $input  = array();
+                $output = array();
+
+                // Double quote after a space or ([{< becomes &#8220; (opening_quote) if not followed by spaces
+
+                $input[]  = 'word "word word';
+                $output[] = 'word &#8220;word word';
+
+                $input[]  = 'word ("word word';
+                $output[] = 'word (&#8220;word word';
+
+                $input[]  = 'word ["word word';
+                $output[] = 'word [&#8220;word word';
+
+                $input[]  = 'word <"word word'; // Invalid HTML input?
+                $output[] = 'word <&#8220;word word';
+
+                $input[]  = 'word &lt;"word word'; // Valid HTML input triggers the closing quote pattern
+                $output[] = 'word &lt;&#8221;word word';
+
+                $input[]  = 'word {"word word';
+                $output[] = 'word {&#8220;word word';
+
+                $input[]  = '"word word';
+                $output[] = '&#8220;word word';
+
+                $input[]  = 'word("word word';
+                $output[] = 'word(&#8220;word word';
+
+                $input[]  = 'word["word word';
+                $output[] = 'word[&#8220;word word';
+
+                $input[]  = 'word<"word word'; // Invalid HTML input?
+                $output[] = 'word<&#8220;word word';
+
+                $input[]  = 'word&lt;"word word'; // Valid HTML input triggers the closing quote pattern
+                $output[] = 'word&lt;&#8221;word word';
+
+                $input[]  = 'word{"word word';
+                $output[] = 'word{&#8220;word word';
+
+                $input[]  = 'word "99 word';
+                $output[] = 'word &#8220;99 word';
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_closing_quote() {
+                $input  = array();
+                $output = array();
+
+                // Double quote becomes &#8221; (closing_quote) unless it is already converted to double_prime or opening_quote.
+
+                $input[]  = 'word word" word';
+                $output[] = 'word word&#8221; word';
+
+                $input[]  = 'word word") word';
+                $output[] = 'word word&#8221;) word';
+
+                $input[]  = 'word word"] word';
+                $output[] = 'word word&#8221;] word';
+
+                $input[]  = 'word word"} word';
+                $output[] = 'word word&#8221;} word';
+
+                $input[]  = 'word word"> word'; // Invalid HTML input?
+                $output[] = 'word word&#8221;> word';
+
+                $input[]  = 'word word"&gt; word'; // Valid HTML should work
+                $output[] = 'word word&#8221;&gt; word';
+
+                $input[]  = 'word word"';
+                $output[] = 'word word&#8221;';
+
+                $input[]  = 'word word"word';
+                $output[] = 'word word&#8221;word';
+
+                $input[]  = 'word"word"word';
+                $output[] = 'word&#8221;word&#8221;word';
+
+                $input[]  = 'test sentence".';
+                $output[] = 'test sentence&#8221;.';
+
+                $input[]  = 'test sentence."';
+                $output[] = 'test sentence.&#8221;';
+
+                $input[]  = 'test sentence". word';
+                $output[] = 'test sentence&#8221;. word';
+
+                $input[]  = 'test sentence." word';
+                $output[] = 'test sentence.&#8221; word';
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_closing_single_quote() {
+                $input  = array();
+                $output = array();
+
+                // Single quote followed by a space or a period becomes &#8217; (closing_single_quote)
+
+                $input[]  = "word word' word";
+                $output[] = "word word&#8217; word";
+
+                $input[]  = "word word'. word";
+                $output[] = "word word&#8217;. word";
+
+                $input[]  = "word word'.word";
+                $output[] = "word word&#8217;.word";
+
+                $input[]  = "word word'";
+                $output[] = "word word&#8217;";
+
+                $input[]  = "test sentence'.";
+                $output[] = "test sentence&#8217;.";
+
+                $input[]  = "test sentence.'";
+                $output[] = "test sentence.&#8217;";
+
+                $input[]  = "test sentence'. word";
+                $output[] = "test sentence&#8217;. word";
+
+                $input[]  = "test sentence.' word";
+                $output[] = "test sentence.&#8217; word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_multiplication() {
+                $input  = array();
+                $output = array();
+
+                // 9x9 becomes 9&#215;9
+
+                $input[]  = "9x9";
+                $output[] = "9&#215;9";
+
+                $input[]  = "12x34";
+                $output[] = "12&#215;34";
+
+                $input[]  = "9 x 9";
+                $output[] = "9 x 9";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_ampersand() {
+                $input  = array();
+                $output = array();
+
+                // & always becomes &#038; unless it is followed by # or ;
+
+                $input[]  = "word & word";
+                $output[] = "word &#038; word";
+
+                $input[]  = "word&word";
+                $output[] = "word&#038;word";
+
+                $input[]  = "word &nbsp; word";
+                $output[] = "word &nbsp; word";
+
+                $input[]  = "word &#038; word";
+                $output[] = "word &#038; word";
+
+                $input[]  = "word &# word";
+                $output[] = "word &# word"; // invalid output?
+
+                $input[]  = "word &44; word";
+                $output[] = "word &44; word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_cockney() {
+                $input  = array();
+                $output = array();
+
+                // "cockney" phrases begin with apos instead of opening_single_quote.
+
+                $input[]  = "word 'tain't word";
+                $output[] = "word &#8217;tain&#8217;t word";
+
+                $input[]  = "word 'twere word";
+                $output[] = "word &#8217;twere word";
+
+                $input[]  = "word 'twas word";
+                $output[] = "word &#8217;twas word";
+
+                $input[]  = "word 'tis word";
+                $output[] = "word &#8217;tis word";
+
+                $input[]  = "word 'twill word";
+                $output[] = "word &#8217;twill word";
+
+                $input[]  = "word 'til word";
+                $output[] = "word &#8217;til word";
+
+                $input[]  = "word 'bout word";
+                $output[] = "word &#8217;bout word";
+
+                $input[]  = "word 'nuff word";
+                $output[] = "word &#8217;nuff word";
+
+                $input[]  = "word 'round word";
+                $output[] = "word &#8217;round word";
+
+                $input[]  = "word 'cause word";
+                $output[] = "word &#8217;cause word";
+
+                $input[]  = "word 'em word";
+                $output[] = "word &#8216;em word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_smart_dashes() {
+                $input  = array();
+                $output = array();
+
+                // Smart dashes, etc
+
+                $input[]  = "word --- word";
+                $output[] = "word &#8212; word";
+
+                $input[]  = "word---word";
+                $output[] = "word&#8212;word";
+
+                $input[]  = "word -- word";
+                $output[] = "word &#8212; word";
+
+                $input[]  = "word--word";
+                $output[] = "word&#8211;word";
+
+                $input[]  = "word - word";
+                $output[] = "word &#8211; word";
+
+                $input[]  = "word-word";
+                $output[] = "word-word";
+
+                $input[]  = "word xn&#8211; word";
+                $output[] = "word xn-- word";
+
+                $input[]  = "wordxn&#8211;word";
+                $output[] = "wordxn--word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Check all baseline patterns.  If anything ever changes in wptexturize(), these tests may fail.
+         *
+         * @ticket 22692
+         */
+        function test_misc_static_replacements() {
+                $input  = array();
+                $output = array();
+
+                $input[]  = "word ... word";
+                $output[] = "word &#8230; word";
+
+                $input[]  = "word...word";
+                $output[] = "word&#8230;word";
+
+                $input[]  = "word `` word";
+                $output[] = "word &#8220; word";
+
+                $input[]  = "word``word";
+                $output[] = "word&#8220;word";
+
+                $input[]  = "word '' word";
+                $output[] = "word &#8221; word";
+
+                $input[]  = "word''word";
+                $output[] = "word&#8221;word";
+
+                $input[]  = "word (tm) word";
+                $output[] = "word &#8482; word";
+
+                $input[]  = "word (tm)word";
+                $output[] = "word &#8482;word";
+
+                $input[]  = "word(tm) word";
+                $output[] = "word(tm) word";
+
+                $input[]  = "word(tm)word";
+                $output[] = "word(tm)word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+
+        /**
+         * All of these tests fail against 3.8 but were not yet listed above.
+         *
+         * @ticket 22692
+         */
+        function test_more_bugs() {
+                $input  = array();
+                $output = array();
+
+                // Tests derived from, and contrary to, the baseline patterns above, where output seemed unusual
+
+                $input[]  = "according to our source, '33% of all students scored less than 50' on the test.";
+                $output[] = "according to our source, &#8216;33% of all students scored less than 50&#8217; on the test.";
+
+                $input[]  = 'according to our source, "33% of all students scored less than 50" on the test.';
+                $output[] = 'according to our source, &#8220;33% of all students scored less than 50&#8221; on the test.';
+
+                $input[]  = "word &lt;'test quote'&rt; word";
+                $output[] = "word &lt;&#8216;test quote&#8217;&rt; word";
+
+                $input[]  = 'word &lt;"test quote"&rt; word';
+                $output[] = 'word &lt;&#8220;test quote&#8221;&rt; word';
+
+                $input[]  = "word 'em word";
+                $output[] = "word &#8217;em word";
+
+                $input[]  = "word xn&#8211; word";
+                $output[] = "word xn&#8211; word";
+
+                $input[]  = "wordxn&#8211;word";
+                $output[] = "wordxn&#8211;word";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Multiplication pattern in 3.8 will break HTML that contains matching server names or directories.
+         *
+         * @ticket 4116
+         */
+        function test_multiplication_in_urls() {
+                $input  = array();
+                $output = array();
+
+                $input[]  = "www.4x3.com";
+                $output[] = "www.4x3.com";
+
+                $input[]  = "4x3.domain.com";
+                $output[] = "4x3.domain.com";
+
+                $input[]  = "<a href='http://www.24x7.com/'>http://www.24x7.com/</a>";
+                $output[] = "<a href='http://www.24x7.com/'>http://www.24x7.com/</a>";
+
+                $input[]  = "<a href='http://www.domain.com/24x7/'>http://www.domain.com/24x7/</a>";
+                $output[] = "<a href='http://www.domain.com/24x7/'>http://www.domain.com/24x7/</a>";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Shortcodes and smart quotes might conflict.  Some of these appear to be fixed already.
+         *
+         * @ticket 6969
+         */
+        function test_quotes_in_shortcodes() {
+                $input  = array();
+                $output = array();
+
+                $input[]  = '[caption]<img src="http://localhost/Great_Wave.jpg" alt="Great Wave" title="Great Wave" width="300" height="205" /> The Great Wave[/caption]';
+                $output[] = '[caption]<img src="http://localhost/Great_Wave.jpg" alt="Great Wave" title="Great Wave" width="300" height="205" /> The Great Wave[/caption]';
+
+                $input[]  = '[code lang="php"]$foo = \'bar\';[/code]';
+                $output[] = '[code lang="php"]$foo = \'bar\';[/code]';
+
+                $input[]  = '[foobar]This is how you "quote" things.[/foobar]';
+                $output[] = '[foobar]This is how you "quote" things.[/foobar]';
+
+                $input[]  = "<pre>\n[[php foo=\"bar\"]\necho \"foo\" . 'bar';\n[/php]]\n</pre>";
+                $output[] = "<pre>\n[[php foo=\"bar\"]\necho \"foo\" . 'bar';\n[/php]]\n</pre>";
+
+                $input[]  = "foo[shortcode]\n\"bar\"\n[/shortcode]foo";
+                $output[] = "foo[shortcode]\n\"bar\"\n[/shortcode]foo";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * User requested smart quotes not break when surrounded by dashes.
+         *
+         * @ticket 20342
+         */
+        function test_dashes_around_quotes() {
+                $input  = array();
+                $output = array();
+
+                // These may fail in 3.8
+
+                $input[]  = 'word---"quote"';
+                $output[] = 'word&#8212;&#8220;quote&#8221;';
+
+                $input[]  = 'word--"quote"';
+                $output[] = 'word&#8211;&#8220;quote&#8221;';
+
+                $input[]  = 'word-"quote"';
+                $output[] = 'word-&#8220;quote&#8221;';
+
+                // These should pass already
+
+                $input[]  = '"quote"---word';
+                $output[] = '&#8220;quote&#8221;&#8212;word';
+
+                $input[]  = '"quote"--word';
+                $output[] = '&#8220;quote&#8221;&#8211;word';
+
+                $input[]  = '"quote"-word';
+                $output[] = '&#8220;quote&#8221;-word';
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
+
+        /**
+         * Numbers after apostrophes are ambiguous due to the ambiguity of single quotes.
+         *
+         * @ticket 26850
+         */
+        function test_quotes_starting_with_digits() {
+                $input  = array();
+                $output = array();
+
+                $input[]  = "in '4 years, 3 months,' Obama cut the deficit";
+                $output[] = "in &#8216;4 years, 3 months,&#8217; Obama cut the deficit";
+
+                $input[]  = "in '40 years, 3 months,' Obama cut the deficit";
+                $output[] = "in &#8216;40 years, 3 months,&#8217; Obama cut the deficit";
+
+                $input[]  = "in '400 years, 3 months,' Obama cut the deficit";
+                $output[] = "in &#8216;400 years, 3 months,&#8217; Obama cut the deficit";
+
+                foreach($input as $key => $in) {
+                        $this->assertEquals( $output[$key], wptexturize( $in ) );
+                }
+        }
 }

tests/phpunit/tests/shortcode.php

@@ -373 +373 @@
                 remove_filter( 'shortcode_atts_bartag', array( $this, '_filter_atts2' ), 10, 3 );
         }
 
+        /**
+         * Check that shortcode_unautop() will always recognize spaces around shortcodes.
+         *
+         * @ticket 22692
+         */
+        function test_spaces_around_shortcodes() {
+                $nbsp = "\xC2\xA0";
+
+                $input  = array();
+
+                $input[] = "<p>[gallery ids=\"37,15,11\"]</p>";
+                $input[] = "<p> [gallery ids=\"37,15,11\"] </p>";
+                $input[] = "<p> {$nbsp}[gallery ids=\"37,15,11\"] {$nbsp}</p>";
+                $input[] = "<p> &nbsp;[gallery ids=\"37,15,11\"] &nbsp;</p>";
+
+                $output = "[gallery ids=\"37,15,11\"]";
+
+                foreach($input as $in) {
+                        $this->assertEquals( $output, shortcode_unautop( $in ) );
+                }
+        }
 }