Changeset 17555

Timestamp:

03/25/2011 02:42:20 AM (14 years ago)

Author:

dd32

Message:

First run of introducing Stream-To-File for the WP_HTTP API. Reduces memory consumption during file downloads. Implemented in download_url() for upgraders. Props sivel. See #16236

Location:

trunk

Files:

• wp-admin/includes/file.php (modified) (2 diffs)
• wp-includes/class-http.php (modified) (12 diffs)
• wp-includes/functions.php (modified) (1 diff)

trunk/wp-admin/includes/file.php

@@ -154 +154 @@
 
 /**
- * Determines a writable directory for temporary files.
- * Function's preference is to WP_CONTENT_DIR followed by the return value of <code>sys_get_temp_dir()</code>, before finally defaulting to /tmp/
- *
- * In the event that this function does not find a writable location, It may be overridden by the <code>WP_TEMP_DIR</code> constant in your <code>wp-config.php</code> file.
- *
- * @since 2.5.0
- *
- * @return string Writable temporary directory
- */
-function get_temp_dir() {
-    static $temp;
-    if ( defined('WP_TEMP_DIR') )
-        return trailingslashit(WP_TEMP_DIR);
-
-    if ( $temp )
-        return trailingslashit($temp);
-
-    $temp = WP_CONTENT_DIR . '/';
-    if ( is_dir($temp) && @is_writable($temp) )
-        return $temp;
-
-    if  ( function_exists('sys_get_temp_dir') ) {
-        $temp = sys_get_temp_dir();
-        if ( @is_writable($temp) )
-            return trailingslashit($temp);
-    }
-
-    $temp = ini_get('upload_tmp_dir');
-    if ( is_dir($temp) && @is_writable($temp) )
-        return trailingslashit($temp);
-
-    $temp = '/tmp/';
-    return $temp;
-}
-
-/**
  * Returns a filename of a Temporary unique file.
  * Please note that the calling function must unlink() this itself.
@@ -520 +484 @@
         return new WP_Error('http_no_file', __('Could not create Temporary file.'));
 
-    $handle = @fopen($tmpfname, 'wb');
-    if ( ! $handle )
-        return new WP_Error('http_no_file', __('Could not create Temporary file.'));
-
-    $response = wp_remote_get($url, array('timeout' => $timeout));
-
-    if ( is_wp_error($response) ) {
-        fclose($handle);
-        unlink($tmpfname);
+    $response = wp_remote_get( $url, array( 'timeout' => $timeout, 'stream' => true, 'filename' => $tmpfname ) );
+
+    if ( is_wp_error( $response ) ) {
+        unlink( $tmpfname );
         return $response;
     }
 
     if ( $response['response']['code'] != '200' ){
-        fclose($handle);
-        unlink($tmpfname);
-        return new WP_Error('http_404', trim($response['response']['message']));
-    }
-
-    fwrite($handle, $response['body']);
-    fclose($handle);
+        unlink( $tmpfname );
+        return new WP_Error( 'http_404', trim( $response['response']['message'] ) );
+    }
 
     return $tmpfname;

trunk/wp-includes/class-http.php

@@ -98 +98 @@
             'compress' => false,
             'decompress' => true,
-            'sslverify' => true
+            'sslverify' => true,
+            'stream' => false,
+            'filename' => null
         );
 
@@ -136 +138 @@
         $r['local'] = $homeURL['host'] == $arrURL['host'] || 'localhost' == $arrURL['host'];
         unset( $homeURL );
+
+        // If we are streaming to a file but no filename was given drop it in the WP temp dir
+        // and pick it's name using the basename of the $url
+        if ( $r['stream']  && empty( $r['filename'] ) )
+            $r['filename'] = get_temp_dir() . basename( $url );
+
+        // Force some settings if we are streaming to a file and check for existence and perms of destination directory
+        if ( $r['stream'] ) {
+            $r['blocking'] = true;
+            if ( ! is_writable( dirname( $r['filename'] ) ) )
+                return new WP_Error( 'http_request_failed', __( 'Destination directory for file streaming does not exist or is not writable.' ) );
+        }
 
         if ( is_null( $r['headers'] ) )
@@ -660 +674 @@
 
         $strResponse = '';
-        while ( ! feof($handle) )
-            $strResponse .= fread($handle, 4096);
-
-        fclose($handle);
+        $bodyStarted = false;
+
+        // If streaming to a file setup the file handle
+        if ( $r['stream'] ) {
+            if ( ! WP_DEBUG )
+                $stream_handle = @fopen( $r['filename'], 'w+' );
+            else
+                $stream_handle = fopen( $r['filename'], 'w+' );
+            if ( ! $stream_handle )
+                return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+
+            while ( ! feof($handle) ) {
+                $block = fread( $handle, 4096 );
+                if ( $bodyStarted ) {
+                    fwrite( $stream_handle, $block );
+                } else {
+                    $strResponse .= $block;
+                    if ( strpos( $strResponse, "\r\n\r\n" ) ) {
+                        $process = WP_Http::processResponse( $strResponse );
+                        $bodyStarted = true;
+                        fwrite( $stream_handle, $process['body'] );
+                        unset( $strResponse );
+                        $process['body'] = '';
+                    }
+                }
+            }
+
+            fclose( $stream_handle );
+
+        } else { 
+            while ( ! feof($handle) )
+                $strResponse .= fread( $handle, 4096 );
+
+            $process = WP_Http::processResponse( $strResponse );
+            unset( $strResponse );
+        }
+
+        fclose( $handle );
 
         if ( true === $secure_transport )
             error_reporting($error_reporting);
 
-        $process = WP_Http::processResponse($strResponse);
-        $arrHeaders = WP_Http::processHeaders($process['headers']);
+        $arrHeaders = WP_Http::processHeaders( $process['headers'] );
 
         // Is the response code within the 400 range?
@@ -691 +738 @@
             $process['body'] = WP_Http_Encoding::decompress( $process['body'] );
 
-        return array('headers' => $arrHeaders['headers'], 'body' => $process['body'], 'response' => $arrHeaders['response'], 'cookies' => $arrHeaders['cookies']);
+        return array( 'headers' => $arrHeaders['headers'], 'body' => $process['body'], 'response' => $arrHeaders['response'], 'cookies' => $arrHeaders['cookies'], 'filename' => $r['filename'] );
     }
 
@@ -835 +882 @@
         }
 
-        $strResponse = stream_get_contents($handle);
-        $meta = stream_get_meta_data($handle);
-
-        fclose($handle);
+        if ( $r['stream'] ) {
+            if ( ! WP_DEBUG )
+                $stream_handle = @fopen( $r['filename'], 'w+' );
+            else
+                $stream_handle = fopen( $r['filename'], 'w+' );
+
+            if ( ! $stream_handle )
+                return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+
+            stream_copy_to_stream( $handle, $stream_handle );
+
+            fclose( $stream_handle );
+            $strResponse = '';
+        } else {
+            $strResponse = stream_get_contents( $handle );
+        }
+
+        $meta = stream_get_meta_data( $handle );
+
+        fclose( $handle );
 
         $processedHeaders = array();
@@ -857 +920 @@
             $strResponse = WP_Http_Encoding::decompress( $strResponse );
 
-        return array('headers' => $processedHeaders['headers'], 'body' => $strResponse, 'response' => $processedHeaders['response'], 'cookies' => $processedHeaders['cookies']);
+        return array( 'headers' => $processedHeaders['headers'], 'body' => $strResponse, 'response' => $processedHeaders['response'], 'cookies' => $processedHeaders['cookies'], 'filename' => $r['filename'] );
     }
 
@@ -1007 +1070 @@
             $theBody = http_inflate( $theBody );
 
+        if ( $r['stream'] ) {
+            if ( !WP_DEBUG )
+                $stream_handle = @fopen( $r['filename'], 'w+' );
+            else
+                $stream_handle = fopen( $r['filename'], 'w+' );
+
+            if ( ! $stream_handle )
+                return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+
+            fwrite( $stream_handle, $theBody );
+            fclose( $stream_handle );
+            $theBody = '';
+        }
+
         $theResponse = array();
         $theResponse['code'] = $info['response_code'];
         $theResponse['message'] = get_status_header_desc($info['response_code']);
 
-        return array('headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $theResponse, 'cookies' => $theHeaders['cookies']);
+        return array( 'headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $theResponse, 'cookies' => $theHeaders['cookies'], 'filename' => $r['filename'] );
     }
 
@@ -1037 +1114 @@
  */
 class WP_Http_Curl {
+
+    /**
+     * Temporary header storage for use with streaming to a file.
+     *
+     * @since 3.2.0
+     * @access private
+     * @var string
+     */
+    private $headers;
 
     /**
@@ -1122 +1208 @@
 
         if ( true === $r['blocking'] )
-            curl_setopt( $handle, CURLOPT_HEADER, true );
-        else
-            curl_setopt( $handle, CURLOPT_HEADER, false );
+            curl_setopt( $handle, CURLOPT_HEADERFUNCTION, array( &$this, 'stream_headers' ) );
+
+        curl_setopt( $handle, CURLOPT_HEADER, false );
+
+        // If streaming to a file open a file handle, and setup our curl streaming handler
+        if ( $r['stream'] ) {
+            if ( ! WP_DEBUG )
+                $stream_handle = @fopen( $r['filename'], 'w+' );
+            else
+                $stream_handle = fopen( $r['filename'], 'w+' );
+            if ( ! $stream_handle )
+                return new WP_Error( 'http_request_failed', sprintf( __( 'Could not open handle for fopen() to %s' ), $r['filename'] ) );
+            curl_setopt( $handle, CURLOPT_FILE, $stream_handle );
+        }
 
         // The option doesn't work with safe mode or when open_basedir is set.
@@ -1156 +1253 @@
 
         $theResponse = curl_exec( $handle );
-
-        if ( !empty($theResponse) ) {
-            $headerLength = curl_getinfo($handle, CURLINFO_HEADER_SIZE);
-            $theHeaders = trim( substr($theResponse, 0, $headerLength) );
-            if ( strlen($theResponse) > $headerLength )
-                $theBody = substr( $theResponse, $headerLength );
-            else
-                $theBody = '';
-            if ( false !== strpos($theHeaders, "\r\n\r\n") ) {
-                $headerParts = explode("\r\n\r\n", $theHeaders);
-                $theHeaders = $headerParts[ count($headerParts) -1 ];
-            }
-            $theHeaders = WP_Http::processHeaders($theHeaders);
-        } else {
+        $theBody = '';
+        $theHeaders = WP_Http::processHeaders( $this->headers );
+
+        if ( ! empty($theResponse) && ! is_bool( $theResponse ) ) // is_bool: when using $args['stream'], curl_exec will return (bool)true
+            $theBody = $theResponse;
+
+        // If no response, and It's not a HEAD request with valid headers returned
+        if ( empty($theResponse) && 'HEAD' != $args['method'] && ! empty($this->headers) ) {
             if ( $curl_error = curl_error($handle) )
                 return new WP_Error('http_request_failed', $curl_error);
             if ( in_array( curl_getinfo( $handle, CURLINFO_HTTP_CODE ), array(301, 302) ) )
                 return new WP_Error('http_request_failed', __('Too many redirects.'));
-
-            $theHeaders = array( 'headers' => array(), 'cookies' => array() );
-            $theBody = '';
-        }
+        }
+
+        unset( $this->headers );
 
         $response = array();
@@ -1185 +1275 @@
         curl_close( $handle );
 
+        if ( $r['stream'] )
+            fclose( $stream_handle );
+
         // See #11305 - When running under safe mode, redirection is disabled above. Handle it manually.
-        if ( !empty($theHeaders['headers']['location']) && (ini_get('safe_mode') || ini_get('open_basedir')) && 0 !== $r['_redirection'] ) {
+        if ( ! empty( $theHeaders['headers']['location'] ) && ( ini_get( 'safe_mode' ) || ini_get( 'open_basedir' ) ) && 0 !== $r['_redirection'] ) {
             if ( $r['redirection']-- > 0 ) {
-                return $this->request($theHeaders['headers']['location'], $r);
+                return $this->request( $theHeaders['headers']['location'], $r );
             } else {
-                return new WP_Error('http_request_failed', __('Too many redirects.'));
+                return new WP_Error( 'http_request_failed', __( 'Too many redirects.' ) );
             }
         }
@@ -1197 +1290 @@
             $theBody = WP_Http_Encoding::decompress( $theBody );
 
-        return array('headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $response, 'cookies' => $theHeaders['cookies']);
+        return array( 'headers' => $theHeaders['headers'], 'body' => $theBody, 'response' => $response, 'cookies' => $theHeaders['cookies'], 'filename' => $r['filename'] );
+    }
+
+    /**
+     * Grab the headers of the cURL request
+     *
+     * Each header is sent individually to this callback, so we append to the $header property for temporary storage
+     *
+     * @since 3.2.0
+     * @access private
+     * @return int
+     */
+    private function stream_headers( $handle, $headers ) {
+        $this->headers .= $headers;
+        return strlen( $headers );
     }
 

trunk/wp-includes/functions.php

@@ -2109 +2109 @@
 
     return rtrim($base, '/') . '/' . ltrim($path, '/');
+}
+
+/**
+ * Determines a writable directory for temporary files.
+ * Function's preference is to WP_CONTENT_DIR followed by the return value of <code>sys_get_temp_dir()</code>, before finally defaulting to /tmp/
+ *
+ * In the event that this function does not find a writable location, It may be overridden by the <code>WP_TEMP_DIR</code> constant in your <code>wp-config.php</code> file.
+ *
+ * @since 2.5.0
+ *
+ * @return string Writable temporary directory
+ */
+function get_temp_dir() {
+    static $temp;
+    if ( defined('WP_TEMP_DIR') )
+        return trailingslashit(WP_TEMP_DIR);
+
+    if ( $temp )
+        return trailingslashit($temp);
+
+    $temp = WP_CONTENT_DIR . '/';
+    if ( is_dir($temp) && @is_writable($temp) )
+        return $temp;
+
+    if  ( function_exists('sys_get_temp_dir') ) {
+        $temp = sys_get_temp_dir();
+        if ( @is_writable($temp) )
+            return trailingslashit($temp);
+    }
+
+    $temp = ini_get('upload_tmp_dir');
+    if ( is_dir($temp) && @is_writable($temp) )
+        return trailingslashit($temp);
+
+    $temp = '/tmp/';
+    return $temp;
 }