Changeset 12174

Timestamp:

11/12/2009 04:05:43 PM (15 years ago)

Author:

ryan

Message:

pomo performance improvements. Props nbachiyski. fixes #10165

Location:

trunk/wp-includes/pomo

Files:

• mo.php (modified) (5 diffs)
• streams.php (modified) (3 diffs)

trunk/wp-includes/pomo/mo.php

@@ -3 +3 @@
  * Class for working with MO files
  *
- * @version $Id: mo.php 221 2009-09-07 21:08:21Z nbachiyski $
+ * @version $Id: mo.php 293 2009-11-12 15:43:50Z nbachiyski $
  * @package pomo
  * @subpackage mo
@@ -22 +22 @@
      */
     function import_from_file($filename) {
-        $reader = new POMO_CachedIntFileReader($filename);
-        if (isset($reader->error)) {
-            return false;
-        }
+        $reader = new POMO_FileReader($filename);
+        if (!$reader->is_resource())
+            return false;
         return $this->import_from_reader($reader);
     }
@@ -114 +113 @@
 
     function import_from_reader($reader) {
-        $reader->setEndian('little');
-        $endian = MO::get_byteorder($reader->readint32());
-        if (false === $endian) {
-            return false;
-        }
-        $reader->setEndian($endian);
-
-        $revision = $reader->readint32();
-        $total = $reader->readint32();
-        // get addresses of array of lenghts and offsets for original string and translations
-        $originals_lenghts_addr = $reader->readint32();
-        $translations_lenghts_addr = $reader->readint32();
-
+        $endian_string = MO::get_byteorder($reader->readint32());
+        if (false === $endian_string) {
+            return false;
+        }
+        $reader->setEndian($endian_string);
+
+        $endian = ('big' == $endian_string)? 'N' : 'V';
+
+        $header = $reader->read(24);
+        if ($reader->strlen($header) != 24)
+            return false;
+
+        // parse header
+        $header = unpack("{$endian}revision/{$endian}total/{$endian}originals_lenghts_addr/{$endian}translations_lenghts_addr/{$endian}hash_length/{$endian}hash_addr", $header);
+        if (!is_array($header))
+            return false;
+
+        extract( $header );
+
+        // support revision 0 of MO format specs, only
+        if ($revision != 0)
+            return false;
+
+        // seek to data blocks
         $reader->seekto($originals_lenghts_addr);
-        $originals_lenghts = $reader->readint32array($total * 2); // each of 
-        $reader->seekto($translations_lenghts_addr);
-        $translations_lenghts = $reader->readint32array($total * 2);
-
-        $length = create_function('$i', 'return $i * 2 + 1;');
-        $offset = create_function('$i', 'return $i * 2 + 2;');
-
-        for ($i = 0; $i < $total; ++$i) {
-            $reader->seekto($originals_lenghts[$offset($i)]);
-            $original = $reader->read($originals_lenghts[$length($i)]);
-            $reader->seekto($translations_lenghts[$offset($i)]);
-            $translation = $reader->read($translations_lenghts[$length($i)]);
-            if ('' == $original) {
+
+        // read originals' indices
+        $originals_lengths_length = $translations_lenghts_addr - $originals_lenghts_addr;
+        if ( $originals_lengths_length != $total * 8 )
+            return false;
+
+        $originals = $reader->read($originals_lengths_length);
+        if ( $reader->strlen( $originals ) != $originals_lengths_length )
+            return false;
+
+        // read translations' indices
+        $translations_lenghts_length = $hash_addr - $translations_lenghts_addr;
+        if ( $translations_lenghts_length != $total * 8 )
+            return false;
+
+        $translations = $reader->read($translations_lenghts_length);
+        if ( $reader->strlen( $translations ) != $translations_lenghts_length )
+            return false;
+
+        // transform raw data into set of indices
+        $originals    = $reader->str_split( $originals, 8 );
+        $translations = $reader->str_split( $translations, 8 );
+
+        // skip hash table
+        $strings_addr = $hash_addr + $hash_length * 4;
+
+        $reader->seekto($strings_addr);
+
+        $strings = $reader->read_all();
+        $reader->close();
+
+        for ( $i = 0; $i < $total; $i++ ) {
+            $o = unpack( "{$endian}length/{$endian}pos", $originals[$i] );
+            $t = unpack( "{$endian}length/{$endian}pos", $translations[$i] );
+            if ( !$o || !$t ) return false;
+
+            // adjust offset due to reading strings to separate space before
+            $o['pos'] -= $strings_addr;
+            $t['pos'] -= $strings_addr;
+
+            $original    = $reader->substr( $strings, $o['pos'], $o['length'] );
+            $translation = $reader->substr( $strings, $t['pos'], $t['length'] );
+
+            if ('' === $original) {
                 $this->set_headers($this->make_headers($translation));
             } else {
-                $this->add_entry($this->make_entry($original, $translation));
+                $entry = &$this->make_entry($original, $translation);
+                $this->entries[$entry->key()] = &$entry;
             }
         }
@@ -150 +192 @@
 
     /**
+     * Build a Translation_Entry from original string and translation strings,
+     * found in a MO file
+     * 
      * @static
+     * @param string $original original string to translate from MO file. Might contain
+     *  0x04 as context separator or 0x00 as singular/plural separator
+     * @param string $translation translation string from MO file. Might contain
+     *  0x00 as a plural translations separator
      */
     function &make_entry($original, $translation) {
-        $args = array();
+        $entry = & new Translation_Entry();
         // look for context
         $parts = explode(chr(4), $original);
         if (isset($parts[1])) {
             $original = $parts[1];
-            $args['context'] = $parts[0];
+            $entry->context = $parts[0];
         }
         // look for plural original
         $parts = explode(chr(0), $original);
-        $args['singular'] = $parts[0];
+        $entry->singular = $parts[0];
         if (isset($parts[1])) {
-            $args['plural'] = $parts[1];
+            $entry->is_plural = true;
+            $entry->plural = $parts[1];
         }
         // plural translations are also separated by \0
-        $args['translations'] = explode(chr(0), $translation);
-        $entry = & new Translation_Entry($args);
+        $entry->translations = explode(chr(0), $translation);
         return $entry;
     }
@@ -179 +228 @@
         return $this->_nplurals;
     }
-
-
 }
 endif;

trunk/wp-includes/pomo/streams.php

@@ -4 +4 @@
  * Based on the classes from Danilo Segan <danilo@kvota.net>
  *
- * @version $Id: streams.php 223 2009-09-07 21:20:13Z nbachiyski $
+ * @version $Id: streams.php 293 2009-11-12 15:43:50Z nbachiyski $
  * @package pomo
  * @subpackage streams
  */
 
-
-if ( !class_exists( 'POMO_StringReader' ) ):
-/**
- * Provides file-like methods for manipulating a string instead
- * of a physical file.
- */
-class POMO_StringReader {
-  var $_pos;
-  var $_str;
-
-    function POMO_StringReader($str = '') {
-        $this->_str = $str;
+if ( !class_exists( 'POMO_Reader' ) ):
+class POMO_Reader {
+    
+    var $endian = 'little';
+    var $_post = '';
+    
+    function POMO_Reader() {
+        $this->is_overloaded = ((ini_get("mbstring.func_overload") & 2) != 0) && function_exists('mb_substr');
         $this->_pos = 0;
-        $this->is_overloaded = ((ini_get("mbstring.func_overload") & 2) != 0) && function_exists('mb_substr');
-    }
-
-    function _substr($string, $start, $length) {
-        if ($this->is_overloaded) {
-            return mb_substr($string,$start,$length,'ascii');
-        } else {
-            return substr($string,$start,$length);
-        }
-    }
-    
-    function _strlen($string) {
-        if ($this->is_overloaded) {
-            return mb_strlen($string,'ascii');
-        } else {
-            return strlen($string);
-        }
-    }
-
-    function read($bytes) {
-        $data = $this->_substr($this->_str, $this->_pos, $bytes);
-        $this->_pos += $bytes;
-        if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str);
-        return $data;
-    }
-
-    function seekto($pos) {
-        $this->_pos = $pos;
-        if ($this->_strlen($this->_str) < $this->_pos) $this->_pos = $this->_strlen($this->_str);
-        return $this->_pos;
-    }
-
-    function pos() {
-        return $this->_pos;
-    }
-
-    function length() {
-        return $this->_strlen($this->_str);
-    }
-
-}
-endif;
-
-if ( !class_exists( 'POMO_CachedFileReader' ) ):
-/**
- * Reads the contents of the file in the beginning.
- */
-class POMO_CachedFileReader extends POMO_StringReader {
-    function POMO_CachedFileReader($filename) {
-        parent::POMO_StringReader();
-        $this->_str = file_get_contents($filename);
-        if (false === $this->_str)
-            return false;
-        $this->_pos = 0;
-    }
-}
-endif;
-
-if ( !class_exists( 'POMO_CachedIntFileReader' ) ):
-/**
- * Allows reading integers from a file.
- */
-class POMO_CachedIntFileReader extends POMO_CachedFileReader {
-
-    var $endian = 'little';
-
-    /**
-     * Opens a file and caches it.
-     *
-     * @param $filename string name of the file to be opened
-     * @param $endian string endianness of the words in the file, allowed
-     *  values are 'little' or 'big'. Default value is 'little'
-     */
-    function POMO_CachedIntFileReader($filename, $endian = 'little') {
-        $this->endian = $endian;
-        parent::POMO_CachedFileReader($filename);
-    }
-
+    }
+    
     /**
      * Sets the endianness of the file.
@@ -117 +37 @@
     function readint32() {
         $bytes = $this->read(4);
-        if (4 != $this->_strlen($bytes))
+        if (4 != $this->strlen($bytes))
             return false;
         $endian_letter = ('big' == $this->endian)? 'N' : 'V';
@@ -133 +53 @@
     function readint32array($count) {
         $bytes = $this->read(4 * $count);
-        if (4*$count != $this->_strlen($bytes))
+        if (4*$count != $this->strlen($bytes))
             return false;
         $endian_letter = ('big' == $this->endian)? 'N' : 'V';
         return unpack($endian_letter.$count, $bytes);
     }
-}
-endif;
+    
+    
+    function substr($string, $start, $length) {
+        if ($this->is_overloaded) {
+            return mb_substr($string, $start, $length, 'ascii');
+        } else {
+            return substr($string, $start, $length);
+        }
+    }
+    
+    function strlen($string) {
+        if ($this->is_overloaded) {
+            return mb_strlen($string, 'ascii');
+        } else {
+            return strlen($string);
+        }
+    }
+    
+    function str_split($string, $chunk_size) {
+        if (!function_exists('str_split')) {
+            $length = $this->strlen($string);
+            $out = array();
+            for ($i = 0; $i < $length; $i += $chunk_size)
+                $out[] = $this->substr($string, $i, $chunk_size);
+            return $out;
+        } else {
+            return str_split( $string, $chunk_size );
+        }
+    }
+    
+        
+    function pos() {
+        return $this->_pos;
+    }
+
+    function is_resource() {
+        return true;
+    }
+    
+    function close() {
+        return true;
+    }
+}
+endif;
+
+if ( !class_exists( 'POMO_FileReader' ) ):
+class POMO_FileReader extends POMO_Reader {
+    function POMO_FileReader($filename) {
+        parent::POMO_Reader();
+        $this->_f = fopen($filename, 'r');
+    }
+    
+    function read($bytes) {
+        return fread($this->_f, $bytes);
+    }
+    
+    function seekto($pos) {
+        if ( -1 == fseek($this->_f, $pos, SEEK_SET)) {
+            return false;
+        }
+        $this->_pos = $pos;
+        return true;
+    }
+    
+    function is_resource() {
+        return is_resource($this->_f);
+    }
+    
+    function feof() {
+        return feof($this->_f);
+    }
+    
+    function close() {
+        return fclose($this->_f);
+    }
+    
+    function read_all() {
+        $all = '';
+        while ( !$this->feof() )
+            $all .= $this->read(4096);
+        return $all;
+    }
+}
+endif;
+
+if ( !class_exists( 'POMO_StringReader' ) ):
+/**
+ * Provides file-like methods for manipulating a string instead
+ * of a physical file.
+ */
+class POMO_StringReader extends POMO_Reader {
+    
+    var $_str = '';
+    
+    function POMO_StringReader($str = '') {
+        parent::POMO_Reader();
+        $this->_str = $str;
+        $this->_pos = 0;
+    }
+
+
+    function read($bytes) {
+        $data = $this->substr($this->_str, $this->_pos, $bytes);
+        $this->_pos += $bytes;
+        if ($this->strlen($this->_str) < $this->_pos) $this->_pos = $this->strlen($this->_str);
+        return $data;
+    }
+
+    function seekto($pos) {
+        $this->_pos = $pos;
+        if ($this->strlen($this->_str) < $this->_pos) $this->_pos = $this->strlen($this->_str);
+        return $this->_pos;
+    }
+
+    function length() {
+        return $this->strlen($this->_str);
+    }
+
+    function read_all() {
+        return $this->substr($this->_str, $this->_pos, $this->strlen($this->_str));
+    }
+    
+}
+endif;
+
+if ( !class_exists( 'POMO_CachedFileReader' ) ):
+/**
+ * Reads the contents of the file in the beginning.
+ */
+class POMO_CachedFileReader extends POMO_StringReader {
+    function POMO_CachedFileReader($filename) {
+        parent::POMO_StringReader();
+        $this->_str = file_get_contents($filename);
+        if (false === $this->_str)
+            return false;
+        $this->_pos = 0;
+    }
+}
+endif;
+
+if ( !class_exists( 'POMO_CachedIntFileReader' ) ):
+/**
+ * Reads the contents of the file in the beginning.
+ */
+class POMO_CachedIntFileReader extends POMO_CachedFileReader {
+    function POMO_CachedIntFileReader($filename) {
+        parent::POMO_CachedFileReader($filename);
+    }
+}
+endif;