Changeset 12174

Timestamp:

11/12/2009 04:05:43 PM (17 years ago)

Author:

ryan

Message:

pomo performance improvements. Props nbachiyski. fixes #10165

Location:

trunk/wp-includes/pomo

Files:

• mo.php (modified) (5 diffs)
• streams.php (modified) (3 diffs)

trunk/wp-includes/pomo/mo.php

@@ -3 +3 @@
  * Class for working with MO files
  *
- * @version $Id: mo.php 221 2009-09-07 21:08:21Z nbachiyski $
+ * @version $Id: mo.php 293 2009-11-12 15:43:50Z nbachiyski $
  * @package pomo
  * @subpackage mo
@@ -22 +22 @@
          */
         function import_from_file($filename) {
-                $reader = new POMO_CachedIntFileReader($filename);
-                if (isset($reader->error)) {
-                        return false;
-                }
+                $reader = new POMO_FileReader($filename);
+                if (!$reader->is_resource())
+                        return false;
                 return $this->import_from_reader($reader);
         }
@@ -114 +113 @@
 
         function import_from_reader($reader) {
-                $reader->setEndian('little');
-                $endian = MO::get_byteorder($reader->readint32());
-                if (false === $endian) {
-                        return false;
-                }
-                $reader->setEndian($endian);
-
-                $revision = $reader->readint32();
-                $total = $reader->readint32();
-                // get addresses of array of lenghts and offsets for original string and translations
-                $originals_lenghts_addr = $reader->readint32();
-                $translations_lenghts_addr = $reader->readint32();
-
+                $endian_string = MO::get_byteorder($reader->readint32());
+                if (false === $endian_string) {
+                        return false;
+                }
+                $reader->setEndian($endian_string);
+
+                $endian = ('big' == $endian_string)? 'N' : 'V';
+
+                $header = $reader->read(24);
+                if ($reader->strlen($header) != 24)
+                        return false;
+
+                // parse header
+                $header = unpack("{$endian}revision/{$endian}total/{$endian}originals_lenghts_addr/{$endian}translations_lenghts_addr/{$endian}hash_length/{$endian}hash_addr", $header);
+                if (!is_array($header))
+                        return false;
+
+                extract( $header );
+
+                // support revision 0 of MO format specs, only
+                if ($revision != 0)
+                        return false;
+
+                // seek to data blocks
                 $reader->seekto($originals_lenghts_addr);
-                $originals_lenghts = $reader->readint32array($total * 2); // each of 
-                $reader->seekto($translations_lenghts_addr);
-                $translations_lenghts = $reader->readint32array($total * 2);
-
-                $length = create_function('$i', 'return $i * 2 + 1;');
-                $offset = create_function('$i', 'return $i * 2 + 2;');
-
-                for ($i = 0; $i < $total; ++$i) {
-                        $reader->seekto($originals_lenghts[$offset($i)]);
-                        $original = $reader->read($originals_lenghts[$length($i)]);
-                        $reader->seekto($translations_lenghts[$offset($i)]);
-                        $translation = $reader->read($translations_lenghts[$length($i)]);
-                        if ('' == $original) {
+
+                // read originals' indices
+                $originals_lengths_length = $translations_lenghts_addr - $originals_lenghts_addr;
+                if ( $originals_lengths_length != $total * 8 )
+                        return false;
+
+                $originals = $reader->read($originals_lengths_length);
+                if ( $reader->strlen( $originals ) != $originals_lengths_length )
+                        return false;
+
+                // read translations' indices
+                $translations_lenghts_length = $hash_addr - $translations_lenghts_addr;
+                if ( $translations_lenghts_length != $total * 8 )
+                        return false;
+
+                $translations = $reader->read($translations_lenghts_length);
+                if ( $reader->strlen( $translations ) != $translations_lenghts_length )
+                        return false;
+
+                // transform raw data into set of indices
+                $originals    = $reader->str_split( $originals, 8 );
+                $translations = $reader->str_split( $translations, 8 );
+
+                // skip hash table
+                $strings_addr = $hash_addr + $hash_length * 4;
+
+                $reader->seekto($strings_addr);
+
+                $strings = $reader->read_all();
+                $reader->close();
+
+                for ( $i = 0; $i < $total; $i++ ) {
+                        $o = unpack( "{$endian}length/{$endian}pos", $originals[$i] );
+                        $t = unpack( "{$endian}length/{$endian}pos", $translations[$i] );
+                        if ( !$o || !$t ) return false;
+
+                        // adjust offset due to reading strings to separate space before
+                        $o['pos'] -= $strings_addr;
+                        $t['pos'] -= $strings_addr;
+
+                        $original    = $reader->substr( $strings, $o['pos'], $o['length'] );
+                        $translation = $reader->substr( $strings, $t['pos'], $t['length'] );
+
+                        if ('' === $original) {
                                 $this->set_headers($this->make_headers($translation));
                         } else {
-                                $this->add_entry($this->make_entry($original, $translation));
+                                $entry = &$this->make_entry($original, $translation);
+                                $this->entries[$entry->key()] = &$entry;
                         }
                 }
@@ -150 +192 @@
 
         /**
+         * Build a Translation_Entry from original string and translation strings,
+         * found in a MO file
+         * 
          * @static
+         * @param string $original original string to translate from MO file. Might contain
+         *      0x04 as context separator or 0x00 as singular/plural separator
+         * @param string $translation translation string from MO file. Might contain
+         *      0x00 as a plural translations separator
          */
         function &make_entry($original, $translation) {
-                $args = array();
+                $entry = & new Translation_Entry();
                 // look for context
                 $parts = explode(chr(4), $original);
                 if (isset($parts[1])) {
                         $original = $parts[1];
-                        $args['context'] = $parts[0];
+                        $entry->context = $parts[0];
                 }
                 // look for plural original
                 $parts = explode(chr(0), $original);
-                $args['singular'] = $parts[0];
+                $entry->singular = $parts[0];
                 if (isset($parts[1])) {
-                        $args['plural'] = $parts[1];
+                        $entry->is_plural = true;
+                        $entry->plural = $parts[1];
                 }
                 // plural translations are also separated by \0
-                $args['translations'] = explode(chr(0), $translation);
-                $entry = & new Translation_Entry($args);
+                $entry->translations = explode(chr(0), $translation);
                 return $entry;
         }
@@ -179 +228 @@
                 return $this->_nplurals;
         }
-
-
 }
 endif;

trunk/wp-includes/pomo/streams.php

@@ -4 +4 @@
  * Based on the classes from Danilo Segan <danilo@kvota.net>
  *
- * @version $Id: streams.php 223 2009-09-07 21:20:13Z nbachiyski $
+ * @version $Id: streams.php 293 2009-11-12 15:43:50Z nbachiyski $
  * @package pomo
  * @subpackage streams
  */
 
-
-if ( !class_exists( 'POMO_StringReader' ) ):
-/**
- * Provides file-like methods for manipulating a string instead
- * of a physical file.
- */
-class POMO_StringReader {
-  var $_pos;
-  var $_str;
-
-        function POMO_StringReader($str = '') {
-                $this->_str = $str;
+if ( !class_exists( 'POMO_Reader' ) ):
+class POMO_Reader {
+        
+        var $endian = 'little';
+        var $_post = '';
+        
+        function POMO_Reader() {
+                $this->is_overloaded = ((ini_get("mbstring.func_overload") & 2) != 0) && function_exists('mb_substr');
                 $this->_pos = 0;
-                $this->is_overloaded = ((ini_get("mbstring.func_overload") & 2) != 0) && function_exists('mb_substr');
-        }
-
-        function _substr($string, $start, $length) {
-                if ($this->is_overloaded) {
-                        return mb_substr($string,$start,$length,'ascii');
-                } else {
-                        return substr($string,$start,$length);
-                }
-        }
-        
-        function _strlen($string) {
-                if ($this->is_overloaded) {
-                        return mb_strlen($string,'ascii');
-                } else {
-                        return strlen($string);
-                }
-        }
-
-        function read($bytes) {
-                $data = $this->_substr($this->_str, $this->_pos, $bytes);
-                $this->_pos += $bytes;
-                if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str);
-                return $data;
-        }
-
-        function seekto($pos) {
-                $this->_pos = $pos;
-                if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str);
-                return $this->_pos;
-        }
-
-        function pos() {
-                return $this->_pos;
-        }
-
-        function length() {
-                return $this->_strlen($this->_str);
-        }
-
-}
-endif;
-
-if ( !class_exists( 'POMO_CachedFileReader' ) ):
-/**
- * Reads the contents of the file in the beginning.
- */
-class POMO_CachedFileReader extends POMO_StringReader {
-        function POMO_CachedFileReader($filename) {
-                parent::POMO_StringReader();
-                $this->_str = file_get_contents($filename);
-                if (false === $this->_str)
-                        return false;
-                $this->_pos = 0;
-        }
-}
-endif;
-
-if ( !class_exists( 'POMO_CachedIntFileReader' ) ):
-/**
- * Allows reading integers from a file.
- */
-class POMO_CachedIntFileReader extends POMO_CachedFileReader {
-
-        var $endian = 'little';
-
-        /**
-         * Opens a file and caches it.
-         *
-         * @param $filename string name of the file to be opened
-         * @param $endian string endianness of the words in the file, allowed
-         *      values are 'little' or 'big'. Default value is 'little'
-         */
-        function POMO_CachedIntFileReader($filename, $endian = 'little') {
-                $this->endian = $endian;
-                parent::POMO_CachedFileReader($filename);
-        }
-
+        }
+        
         /**
          * Sets the endianness of the file.
@@ -117 +37 @@
         function readint32() {
                 $bytes = $this->read(4);
-                if (4 != $this->_strlen($bytes))
+                if (4 != $this->strlen($bytes))
                         return false;
                 $endian_letter = ('big' == $this->endian)? 'N' : 'V';
@@ -133 +53 @@
         function readint32array($count) {
                 $bytes = $this->read(4 * $count);
-                if (4*$count != $this->_strlen($bytes))
+                if (4*$count != $this->strlen($bytes))
                         return false;
                 $endian_letter = ('big' == $this->endian)? 'N' : 'V';
                 return unpack($endian_letter.$count, $bytes);
         }
-}
-endif;
+        
+        
+        function substr($string, $start, $length) {
+                if ($this->is_overloaded) {
+                        return mb_substr($string, $start, $length, 'ascii');
+                } else {
+                        return substr($string, $start, $length);
+                }
+        }
+        
+        function strlen($string) {
+                if ($this->is_overloaded) {
+                        return mb_strlen($string, 'ascii');
+                } else {
+                        return strlen($string);
+                }
+        }
+        
+        function str_split($string, $chunk_size) {
+                if (!function_exists('str_split')) {
+                        $length = $this->strlen($string);
+                        $out = array();
+                        for ($i = 0; $i < $length; $i += $chunk_size)
+                                $out[] = $this->substr($string, $i, $chunk_size);
+                        return $out;
+                } else {
+                        return str_split( $string, $chunk_size );
+                }
+        }
+        
+                
+        function pos() {
+                return $this->_pos;
+        }
+
+        function is_resource() {
+                return true;
+        }
+        
+        function close() {
+                return true;
+        }
+}
+endif;
+
+if ( !class_exists( 'POMO_FileReader' ) ):
+class POMO_FileReader extends POMO_Reader {
+        function POMO_FileReader($filename) {
+                parent::POMO_Reader();
+                $this->_f = fopen($filename, 'r');
+        }
+        
+        function read($bytes) {
+                return fread($this->_f, $bytes);
+        }
+        
+        function seekto($pos) {
+                if ( -1 == fseek($this->_f, $pos, SEEK_SET)) {
+                        return false;
+                }
+                $this->_pos = $pos;
+                return true;
+        }
+        
+        function is_resource() {
+                return is_resource($this->_f);
+        }
+        
+        function feof() {
+                return feof($this->_f);
+        }
+        
+        function close() {
+                return fclose($this->_f);
+        }
+        
+        function read_all() {
+                $all = '';
+                while ( !$this->feof() )
+                        $all .= $this->read(4096);
+                return $all;
+        }
+}
+endif;
+
+if ( !class_exists( 'POMO_StringReader' ) ):
+/**
+ * Provides file-like methods for manipulating a string instead
+ * of a physical file.
+ */
+class POMO_StringReader extends POMO_Reader {
+        
+        var $_str = '';
+        
+        function POMO_StringReader($str = '') {
+                parent::POMO_Reader();
+                $this->_str = $str;
+                $this->_pos = 0;
+        }
+
+
+        function read($bytes) {
+                $data = $this->substr($this->_str, $this->_pos, $bytes);
+                $this->_pos += $bytes;
+                if ($this->strlen($this->_str) < $this->_pos) $this->_pos = $this->strlen($this->_str);
+                return $data;
+        }
+
+        function seekto($pos) {
+                $this->_pos = $pos;
+                if ($this->strlen($this->_str) < $this->_pos) $this->_pos = $this->strlen($this->_str);
+                return $this->_pos;
+        }
+
+        function length() {
+                return $this->strlen($this->_str);
+        }
+
+        function read_all() {
+                return $this->substr($this->_str, $this->_pos, $this->strlen($this->_str));
+        }
+        
+}
+endif;
+
+if ( !class_exists( 'POMO_CachedFileReader' ) ):
+/**
+ * Reads the contents of the file in the beginning.
+ */
+class POMO_CachedFileReader extends POMO_StringReader {
+        function POMO_CachedFileReader($filename) {
+                parent::POMO_StringReader();
+                $this->_str = file_get_contents($filename);
+                if (false === $this->_str)
+                        return false;
+                $this->_pos = 0;
+        }
+}
+endif;
+
+if ( !class_exists( 'POMO_CachedIntFileReader' ) ):
+/**
+ * Reads the contents of the file in the beginning.
+ */
+class POMO_CachedIntFileReader extends POMO_CachedFileReader {
+        function POMO_CachedIntFileReader($filename) {
+                parent::POMO_CachedFileReader($filename);
+        }
+}
+endif;