WordPress.org

Make WordPress Core

Changeset 10538


Ignore:
Timestamp:
02/10/09 19:34:52 (5 years ago)
Author:
azaozz
Message:

Completely New LiveJournal Importer update, props beaulebens, see #8999

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/wp-admin/import/livejournal.php

    r10501 r10538  
    11<?php 
     2 
    23/** 
    34 * LiveJournal API Importer 
     
    910// XML-RPC library for communicating with LiveJournal API 
    1011require_once( ABSPATH . WPINC . '/class-IXR.php' ); 
    11  
    12 // Snoopy for getting comments (with cookies) 
    13 require_once( ABSPATH . WPINC . '/class-snoopy.php' ); 
    1412 
    1513/** 
     
    2725    var $username; 
    2826    var $password; 
    29     var $snoop; 
    3027    var $comment_meta; 
    3128    var $comments; 
    3229    var $usermap; 
    3330    var $postmap; 
     31    var $commentmap; 
    3432    var $pointers = array(); 
    3533     
     
    189187                <input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Continue previous import' ) ) ?>" /> 
    190188            </p> 
    191             <p><a href="<?php echo $_SERVER['PHP_SELF'] . '?import=livejournal&amp;step=-1&amp;_wpnonce=' . wp_create_nonce( 'lj-api-import' ) . '&amp;_wp_http_referer=' . attribute_escape( $_SERVER['REQUEST_URI'] ) ?>"><?php _e( 'Cancel &amp; start a new import' ) ?></a></p> 
     189            <p class="submitbox"><a href="<?php echo $_SERVER['PHP_SELF'] . '?import=livejournal&amp;step=-1&amp;_wpnonce=' . wp_create_nonce( 'lj-api-import' ) . '&amp;_wp_http_referer=' . attribute_escape( $_SERVER['REQUEST_URI'] ) ?>" class="deletion submitdelete"><?php _e( 'Cancel &amp; start a new import' ) ?></a></p> 
    192190            <p> 
    193191        <?php else : ?> 
     
    229227         
    230228            <p><?php _e( '<strong>NOTE:</strong> If the import process is interrupted for <em>any</em> reason, come back to this page and it will continue from where it stopped automatically.' ) ?></p> 
     229 
     230            <noscript> 
     231                <p><?php _e( '<strong>NOTE:</strong> You appear to have JavaScript disabled, so you will need to manually click through each step of this importer. If you enable JavaScript, it will step through automatically.' ) ?></p> 
     232            </noscript> 
    231233        <?php endif; ?> 
    232234        </form> 
     
    249251            $lastsync = date( 'Y-m-d H:i:s', strtotime( get_option( 'ljapi_lastsync' ) ) ); 
    250252            $synclist = $this->lj_ixr( 'syncitems', array( 'ver' => 1, 'lastsync' => $lastsync ) ); 
    251             $this->log( $synclist, 'ljimport-items-' . $total . '.txt' ); 
    252253             
    253254            // Keep track of if we've downloaded everything 
     
    262263                } 
    263264            } 
    264  
    265             update_option( 'ljapi_sync_item_times', $sync_item_times ); 
    266             update_option( 'ljapi_total', $total ); 
    267             update_option( 'ljapi_count', $count ); 
    268             update_option( 'ljapi_lastsync', $lastsync ); 
    269265        } while ( $total > $count ); 
    270266        // endwhile - all post meta is cached locally 
    271         $this->log( $sync_item_times, 'ljimport-post-mod-times.txt' ); 
     267         
     268        update_option( 'ljapi_sync_item_times', $sync_item_times ); 
     269        update_option( 'ljapi_total', $total ); 
     270        update_option( 'ljapi_count', $count ); 
     271        update_option( 'ljapi_lastsync', $lastsync ); 
     272 
     273        echo '<p>' . __( 'Post metadata has been downloaded, proceeding with posts...' ) . '</p>'; 
    272274         
    273275        echo '<ol>'; 
     
    286288                                                            'lineendings' => 'pc', 
    287289                                                            'lastsync' => $lastsync ) ); 
    288             $this->log( $itemlist, 'ljimport-posts-' . $imported_count . '.txt' ); 
    289290            if ( is_wp_error( $itemlist ) ) 
    290291                return $itemlist; 
     292                 
    291293            if ( $num = count( $itemlist['events'] ) ) { 
    292294                foreach ( $itemlist['events'] as $event ) { 
     
    313315         
    314316        $user = wp_get_current_user(); 
    315         $post_author   = $user->ID; 
    316         $post_status   = ( 'private' == trim( $post['security'] ) ) ? 'private' : 'publish'; // Only me 
    317         $post_password = ( 'usemask' == trim( $post['security'] ) ) ? $this->protected_password : ''; // "Friends" via password 
     317        $post_author      = $user->ID; 
     318        $post['security'] = !empty( $post['security'] ) ? $post['security'] : ''; 
     319        $post_status      = ( 'private' == trim( $post['security'] ) ) ? 'private' : 'publish'; // Only me 
     320        $post_password    = ( 'usemask' == trim( $post['security'] ) ) ? $this->protected_password : ''; // "Friends" via password 
    318321 
    319322        // For some reason, LJ sometimes sends a date as "2004-04-1408:38:00" (no space btwn date/time) 
     
    323326         
    324327        // Cleaning up and linking the title 
    325         $post_title = trim( $post['subject'] ); 
     328        $post_title = isset( $post['subject'] ) ? trim( $post['subject'] ) : ''; 
    326329        $post_title = $this->translate_lj_user( $post_title ); // Translate it, but then we'll strip the link 
    327330        $post_title = strip_tags( $post_title ); // Can't have tags in the title in WP 
     
    366369            $postdata['post_ID']   = $post_id; 
    367370            $postdata['lj_itemid'] = $post['itemid']; 
    368             $this->log( $postdata, 'ljimport-post-' . $post_id . '.txt' ); 
    369371             
    370372            // Handle all the metadata for this post 
     
    405407    } 
    406408     
    407     // Loops through and gets comment meta and content from LJ in batches 
    408     // Writes raw XML files to disk for later processing 
    409     function download_comments() { 
     409    // Set up a session (authenticate) with LJ 
     410    function getsession() { 
    410411        // Get a session via XMLRPC 
    411412        $cookie = $this->lj_ixr( 'sessiongenerate', array( 'ver' => 1, 'expiration' => 'short' ) ); 
    412          
    413         // Comment Meta 
     413        return new WP_Http_Cookie( array( 'name' => 'ljsession', 'value' => $cookie['ljsession'] ) ); 
     414    } 
     415     
     416    // Loops through and gets comment meta from LJ in batches 
     417    function download_comment_meta() { 
     418        $cookie = $this->getsession(); 
    414419         
    415420        // Load previous state (if any) 
    416421        $this->usermap = (array) get_option( 'ljapi_usermap' ); 
    417         $maxid         = (int) get_option( 'ljapi_maxid' ) || 1; 
    418         $highest_id    = (int) get_option( 'ljapi_highest_id' ); 
    419  
    420         // Snoopy is required to handle the cookie 
    421         $this->snoop = new Snoopy(); 
    422         $this->snoop->cookies = $cookie; 
     422        $maxid         = get_option( 'ljapi_maxid' ) ? get_option( 'ljapi_maxid' ) : 1; 
     423        $highest_id    = get_option( 'ljapi_highest_id' ) ? get_option( 'ljapi_highest_id' ) : 0; 
    423424         
    424425        // We need to loop over the metadata request until we have it all 
    425426        while ( $maxid > $highest_id ) { 
    426427            // Now get the meta listing 
    427             if ( !$this->snoop->fetch( $this->comments_url . '?get=comment_meta&startid=' . ( $highest_id + 1 ) ) ) 
    428                 return new WP_Error( 'Snoopy', __( 'Failed to retrieve comment meta information from LiveJournal. Please try again soon.' ) ); 
    429  
    430             // Snoopy doesn't provide an accessor for results... 
    431             $results = $this->snoop->results; 
     428            $results = wp_remote_get( $this->comments_url . '?get=comment_meta&startid=' . ( $highest_id + 1 ), 
     429                                        array( 'cookies' => array( $cookie ), 'timeout' => 20 ) ); 
     430            if ( is_wp_error( $results ) ) 
     431                return new WP_Error( 'comment_meta', __( 'Failed to retrieve comment meta information from LiveJournal. Please try again soon.' ) ); 
     432             
     433            $results = wp_remote_retrieve_body( $results ); 
    432434             
    433435            // Get the maxid so we know if we have them all yet 
    434436            preg_match( '|<maxid>(\d+)</maxid>|', $results, $matches ); 
     437            if ( 0 == $matches[1] ) { 
     438                // No comment meta = no comments for this journal 
     439                echo '<p>' . __( 'You have no comments to import!' ) . '</p>'; 
     440                update_option( 'ljapi_highest_id', 1 ); 
     441                update_option( 'ljapi_highest_comment_id', 1 ); 
     442                return false; // Bail out of comment importing entirely 
     443            } 
    435444            $maxid = !empty( $matches[1] ) ? $matches[1] : $maxid; 
    436445             
     
    447456                $this->usermap[$userid] = $matches[2][$count]; // need this in memory for translating ids => names 
    448457                 
    449             update_option( 'ljapi_usermap',    $this->usermap ); 
    450             update_option( 'ljapi_maxid',      $maxid ); 
    451             update_option( 'ljapi_highest_id', $highest_id ); 
     458            wp_cache_flush(); 
    452459        } 
    453460        // endwhile - should have seen all comment meta at this point 
    454          
    455          
    456         // Download Comment XML 
     461 
     462        update_option( 'ljapi_usermap',    $this->usermap ); 
     463        update_option( 'ljapi_maxid',      $maxid ); 
     464        update_option( 'ljapi_highest_id', $highest_id ); 
     465         
     466        echo '<p>' . __( ' Comment metadata downloaded successfully, proceeding with posts...' ) . '</p>'; 
     467         
     468        return true; 
     469    } 
     470     
     471    // Downloads actual comment bodies from LJ 
     472    // Inserts them all directly to the DB, with additional info stored in "spare" fields 
     473    function download_comment_bodies() { 
     474        global $wpdb; 
     475        $cookie = $this->getsession(); 
    457476         
    458477        // Load previous state (if any) 
    459         $highest_id          = (int) get_option( 'ljapi_highest_comment_id' ); 
    460         $comment_xml_files   = get_option( 'ljapi_comment_xml_files' ); 
    461         if ( !is_array( $comment_xml_files ) ) { 
    462             update_option( 'ljapi_comment_xml_files', array() ); 
    463             $comment_xml_files = array(); 
    464         } 
    465          
    466         echo '<ol>'; 
    467          
    468         // And now request the actual comments, and keep going until we have them all 
    469         while ( $maxid > $highest_id ) { 
     478        $this->usermap = (array) get_option( 'ljapi_usermap' ); 
     479        $maxid         = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1; 
     480        $highest_id    = (int) get_option( 'ljapi_highest_comment_id' ); 
     481        $loop = 0; 
     482        while ( $maxid > $highest_id && $loop < 5 ) { // We do 5 loops per call to avoid memory limits 
     483            $loop++; 
     484             
    470485            // Get a batch of comments, using the highest_id we've already got as a starting point 
    471             if ( !$this->snoop->fetch( $this->comments_url . '?get=comment_body&startid=' . ( $highest_id + 1 ) ) ) 
    472                 return new WP_Error( 'Snoopy', __( 'Failed to retrieve comment bodies from LiveJournal. Please try again soon.' ) ); 
    473              
    474             // Get the highest post ID in this batch (required for loop control) 
    475             $results = $this->snoop->results; 
    476             preg_match_all( '|<comment id=\'(\d+)\'|i', $results, $comments ); 
    477             for ( $r = 0; $r < count( $comments[1] ); $r++ ) { 
    478                 if ( $comments[1][$r] > $highest_id ) 
    479                     $highest_id = $comments[1][$r]; 
    480             } 
    481              
    482             // $this->snoop-results is where the actual response is stored 
    483             $this->log( $this->snoop->results, 'ljimport-comment-bodies-' . $highest_id . '.txt' ); 
    484              
    485             // Store in uploads dir. Can't use *.xml because it's not allowed 
    486             $results = wp_upload_bits( 'raw-comments-' . $highest_id . '.txt', null, $results ); 
    487             if ( !empty( $results['error'] ) ) 
    488                 return new WP_Error( 'xml', $results['error'] ); 
    489             $comment_xml_files[] = $results['file']; 
    490              
    491             echo '<li>' . sprintf( __( 'Downloaded <strong>%s</strong>' ), basename( $results['file'] ) ) . '</li>'; 
    492             ob_flush(); flush(); 
    493              
    494             $comment_xml_files = array_unique( $comment_xml_files ); 
    495             update_option( 'ljapi_comment_xml_files', $comment_xml_files ); 
    496             update_option( 'ljapi_comment_xml_files_count', count( $comment_xml_files ) ); 
     486            $results = wp_remote_get( $this->comments_url . '?get=comment_body&startid=' . ( $highest_id + 1 ), 
     487                                        array( 'cookies' => array( $cookie ), 'timeout' => 20 ) ); 
     488            if ( is_wp_error( $results ) ) 
     489                return new WP_Error( 'comment_bodies', __( 'Failed to retrieve comment bodies from LiveJournal. Please try again soon.' ) ); 
     490             
     491            $results = wp_remote_retrieve_body( $results ); 
     492             
     493            // Parse out each comment and insert directly 
     494            preg_match_all( '|<comment id=\'(\d+)\'.*</comment>|iUs', $results, $matches ); 
     495            for ( $c = 0; $c < count( $matches[0] ); $c++ ) { 
     496                // Keep track of highest id seen 
     497                if ( $matches[1][$c] > $highest_id ) { 
     498                    $highest_id = $matches[1][$c]; 
     499                    update_option( 'ljapi_highest_comment_id', $highest_id ); 
     500                } 
     501                     
     502                $comment = $matches[0][$c]; 
     503 
     504                // Filter out any captured, deleted comments (nothing useful to import) 
     505                $comment = preg_replace( '|<comment id=\'\d+\' jitemid=\'\d+\' posterid=\'\d+\' state=\'D\'[^/]*/>|is', '', $comment ); 
     506 
     507                // Parse this comment into an array and insert 
     508                $comment = $this->parse_comment( $comment ); 
     509                $id = wp_insert_comment( $comment ); 
     510 
     511                // Clear cache 
     512                clean_comment_cache( $id ); 
     513            } 
     514             
     515            // Clear cache to preseve memory 
     516            wp_cache_flush(); 
    497517        } 
    498518        // endwhile - all comments downloaded and ready for bulk processing 
    499519         
    500         echo '</ol>'; 
     520        // Counter just used to show progress to user 
     521        update_option( 'ljapi_comment_batch', ( (int) get_option( 'ljapi_comment_batch' ) + 1 ) ); 
    501522         
    502523        return true; 
    503524    } 
    504525     
    505     function parse_comment_xml( $xml_file ) { 
    506         if ( !is_file( $xml_file ) || !is_readable( $xml_file ) ) 
    507             return new WP_Error( 'file', sprintf( __( 'Could not access comment XML file: %s'), $filename ) ); 
    508              
    509         // Get content from file 
    510         $xml = @file_get_contents( $xml_file ); 
    511  
    512         $cache_files = get_option( 'ljapi_comment_cache_files' ); 
    513         if ( !is_array( $cache_files ) ) 
    514             $cache_files = array(); 
    515          
    516         // Parse XML into comments 
    517         preg_match_all( '|<comment id.*</comment>|iUs', $xml, $matches ); 
    518         unset( $xml ); 
    519         for ( $c = 0; $c < count( $matches[0] ); $c++ ) { 
    520             $comment = $matches[0][$c]; 
    521              
    522             // Filter out any captured, deleted comments (nothing useful to import) 
    523             $comment = preg_replace( '|<comment id=\'\d+\' jitemid=\'\d+\' posterid=\'\d+\' state=\'D\'[^/]*/>|is', '', $comment ); 
    524              
    525             // Parse this comment into an array 
    526             $comment = $this->parse_comment( $comment ); 
    527             if ( empty( $comment['comment_post_ID'] ) ) 
    528                 continue; 
    529              
    530             // Add this comment to the appropriate cache file 
    531             $filename = $this->full_path( 'ljimport-comments-' . $comment['comment_post_ID'] . '.php' ); 
    532             if ( $this->write_file( '<?php $comments[] = ' . var_export( $comment, true ) . '; ?>' . "\n",  
    533                                 $filename,  
    534                                 $comment['comment_post_ID'],  
    535                                 'a' ) ) 
    536             { 
    537                 // Keep track of files used 
    538                 $cache_files[] = $filename; 
    539             } 
    540         } 
    541          
    542         // Update list of files in the DB 
    543         sort( $cache_files ); 
    544         $cache_files = array_unique( $cache_files ); 
    545         update_option( 'ljapi_comment_cache_files', $cache_files ); 
    546         update_option( 'ljapi_comment_cache_files_count', count( $cache_files ) ); 
    547         $this->close_file_pointers(); 
    548          
    549         // Don't need this XML file anymore 
    550         unlink( $xml_file ); 
    551          
    552         return true; 
    553     } 
    554      
     526    // Takes a block of XML and parses out all the elements of the comment 
    555527    function parse_comment( $comment ) { 
    556528        global $wpdb; 
     
    563535        $lj_comment_post_ID = $matches[1]; 
    564536        preg_match( '| posterid=\'(\d+)\'|i', $attribs[1], $matches ); 
    565         $comment_author_ID = $matches[1]; 
    566         preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches ); 
    567         $lj_comment_parent = $matches[1]; 
    568         preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches ); 
    569         $lj_comment_state = !empty( $matches[1] ) ? $matches[1] : 'A'; 
     537        $comment_author_ID = isset( $matches[1] ) ? $matches[1] : 0; 
     538        preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches ); // optional 
     539        $lj_comment_parent = isset( $matches[1] ) ? $matches[1] : 0; 
     540        preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches ); // optional 
     541        $lj_comment_state = isset( $matches[1] ) ? $matches[1] : 'A'; 
    570542         
    571543        // Clean up "subject" - this will become the first line of the comment in WP 
    572544        preg_match( '|<subject>(.*)</subject>|is', $comment, $matches ); 
    573         $comment_subject = $wpdb->escape( trim( $matches[1] ) ); 
    574         if ( 'Re:' == $comment_subject ) 
    575             $comment_subject = ''; 
     545        if ( isset( $matches[1] ) ) { 
     546            $comment_subject = $wpdb->escape( trim( $matches[1] ) ); 
     547            if ( 'Re:' == $comment_subject ) 
     548                $comment_subject = ''; 
     549        } 
    576550         
    577551        // Get the body and HTMLize it 
     
    591565         
    592566        // Grab IP if available 
    593         preg_match( '|<property name=\'poster_ip\'>(.*)</property>|i', $comment, $matches ); 
    594         $comment_author_IP = $matches[1]; 
     567        preg_match( '|<property name=\'poster_ip\'>(.*)</property>|i', $comment, $matches ); // optional 
     568        $comment_author_IP = isset( $matches[1] ) ? $matches[1] : ''; 
    595569         
    596570        // Try to get something useful for the comment author, especially if it was "my" comment 
    597         $author = ( substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' || empty( $comment_author_ID ) ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID]; 
     571        $author = ( empty( $comment_author_ID ) || empty( $this->usermap[$comment_author_ID] ) || substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID]; 
    598572        if ( get_option( 'ljapi_username' ) == $author ) { 
    599573            $user    = wp_get_current_user(); 
     
    614588                        'comment_author' => $author, 
    615589                        'comment_author_url' => $url, 
     590                        'comment_author_email' => '', 
    616591                        'comment_content' => $comment_content, 
    617592                        'comment_date' => $comment_date, 
    618593                        'comment_author_IP' => ( !empty( $comment_author_IP ) ? $comment_author_IP : '' ), 
    619594                        'comment_approved' => ( in_array( $lj_comment_state, array( 'A', 'F' ) ) ? 1 : 0 ), 
    620                         'comment_agent' => 'WP LJ Importer', 
    621                         'user_id' => $user_id 
    622                         ); 
     595                        'comment_karma' => $lj_comment_ID, // Need this and next value until rethreading is done 
     596                        'comment_agent' => $lj_comment_parent, 
     597                        'comment_type' => 'livejournal',  // Custom type, so we can find it later for processing 
     598                        'user_ID' => $user_id 
     599                    ); 
    623600    } 
    624601     
     
    628605        global $wpdb; 
    629606        if ( empty( $this->postmap[$post] ) ) 
    630             $this->postmap[$post] = $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) ); 
     607            $this->postmap[$post] = (int) $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) ); 
    631608        return $this->postmap[$post]; 
    632609    } 
    633610     
    634     // Re-build the threading within a single cache file 
    635     function thread_comments( $filename ) { 
    636         if ( !is_file( $filename ) || !is_readable( $filename ) ) 
    637             return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) ); 
    638          
    639         $comments = array(); 
    640         @include( $filename ); 
    641         $this->comments = $comments; 
    642         unset( $comments ); 
    643         if ( !is_array( $this->comments ) ) 
    644             $this->comments = array(); 
    645              
    646         $count = count( $this->comments ); 
    647         for ( $c = 0; $c < $count; $c++ ) { 
    648             // Skip anything that's not "top-level" for now 
    649             if ( 0 != $this->comments[$c]['lj_comment_parent'] ) 
    650                 continue; 
    651             $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] ); 
    652         } 
    653          
    654         // Remove anything that's not supposed to be at top level 
    655         $top_comments = array(); 
    656         for ( $c = 0; $c < $count; $c++ ) { 
    657             if ( 0 == $this->comments[$c]['lj_comment_parent'] ) { 
    658                 $top_comments[] = $this->comments[$c]; 
    659             } 
    660         } 
    661          
    662         // Write back to file 
    663         @unlink( $filename ); 
    664         $this->write_file( '<?php $comments = ' . var_export( $top_comments, true ) . '; ?>', $filename, $count, 'w' ); 
    665         unset( $top_comments ); 
    666         $this->close_file_pointers(); 
    667          
    668         // Reference this file as being threaded 
    669         $files = get_option( 'ljapi_comment_threaded_files' ); 
    670         $files[] = $filename; 
    671         array_unique( $files ); 
    672         update_option( 'ljapi_comment_threaded_files', $files ); 
    673         update_option( 'ljapi_comment_threaded_files_count', count( $files ) ); 
    674          
    675         return true; 
    676     } 
    677      
    678     function get_child_comments( $id ) { 
    679         $children = array(); 
    680         $count = count( $this->comments ); 
    681         for ( $c = 0; $c < $count; $c++ ) { 
    682             // This comment is a child of the $id 
    683             if ( $id == $this->comments[$c]['lj_comment_parent'] ) { 
    684                 $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] ); 
    685                 $children[] = $this->comments[$c]; 
    686             } 
    687         } 
    688         return $children; 
    689     } 
    690      
    691     // Inserts the contents of each cache file (should be threaded already) 
    692     function insert_comments( $filename ) { 
    693         echo '<ol>'; 
    694  
    695         if ( !is_file( $filename ) || !is_readable( $filename ) ) 
    696             return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) ); 
    697          
    698         $comments = array(); 
    699         @include( $filename ); 
    700         $this->comments = $comments; 
    701         unset( $comments ); 
    702         if ( !is_array( $this->comments ) ) 
    703             $this->comments = array(); 
    704              
    705         $count = count( $this->comments ); 
    706         for ( $c = 0; $c < $count; $c++ ) { 
    707             $comment =& $this->comments[$c]; 
    708             echo '<li>'; 
    709             printf( __( 'Imported comment from <strong>%s</strong> on %s' ), $comment['comment_author'], $comment['comment_date'] ); 
    710  
    711             $id = wp_insert_comment( $comment ); 
    712             $comment['comment_ID'] = $id; 
    713             if ( count( $comment['children'] ) ) { 
    714                 _e( ' and replies:' ); 
    715                 $this->insert_child_comments( $comment['children'], $id ); 
    716             } 
    717              
    718             echo '</li>'; 
    719         } 
    720          
    721         // Remove the file now that we're done with it 
    722         @unlink( $filename ); 
    723  
    724         echo '</ol>'; 
    725          
    726         return true; 
    727     } 
    728      
    729     function insert_child_comments( &$comments, $parent ) { 
    730         echo '<ol>'; 
    731         $count = count( $comments ); 
    732         for ( $c = 0; $c < $count; $c++ ) { 
    733             $comment =& $comments[$c]; 
    734             $comment['comment_parent'] = $parent; 
    735             echo '<li>'; 
    736             printf( __( 'Imported reply from <strong>%s</strong> on %s' ), $comment['comment_author'], $comment['comment_date'] ); 
    737  
    738             $id = wp_insert_comment( $comment ); 
    739             $comment['comment_ID'] = $id; 
    740             if ( count( $comment['children'] ) ) { 
    741                 _e( ' and replies:' ); 
    742                 $this->insert_child_comments( $comment['children'], $id ); 
    743             } 
    744              
    745             echo '</li>'; 
    746         } 
    747         echo '</ol>'; 
     611    // Gets the comment_ID that a LJ comment has been saved as within WP 
     612    function get_wp_comment_ID( $comment ) { 
     613        global $wpdb; 
     614        if ( empty( $this->commentmap[$comment] ) ) 
     615            $this->commentmap[$comment] = $wpdb->get_var( $wpdb->prepare( "SELECT comment_ID FROM $wpdb->comments WHERE comment_karma = %d", $comment ) ); 
     616        return $this->commentmap[$comment]; 
    748617    } 
    749618             
     
    768637            return $this->ixr->getResponse(); 
    769638        } else { 
    770             $this->log( $this->ixr->message, 'ljimport-error-' . $method . '.txt' ); 
    771639            return new WP_Error( 'IXR', __( 'XML-RPC Request Failed - ' ) . $this->ixr->getErrorCode() . ': ' . $this->ixr->getErrorMessage() ); 
    772640        } 
     
    789657                break; 
    790658            case 1 : 
    791             case 2 : 
    792659                $this->ixr = new IXR_Client( $this->ixr_url ); 
    793660                // Intentional no break 
     661            case 2 : 
    794662            case 3 : 
    795             case 4 : 
    796             case 5 : 
    797663                check_admin_referer( 'lj-api-import' ); 
    798664                $result = $this->{ 'step' . $step }(); 
     
    838704        if ( is_wp_error( $login ) ) { 
    839705            if ( 100 == $this->ixr->getErrorCode() || 101 == $this->ixr->getErrorCode() ) { 
     706                delete_option( 'ljapi_username' ); 
     707                delete_option( 'ljapi_password' ); 
    840708                ?> 
    841709                <p><?php _e( 'Logging in to LiveJournal failed. Check your username and password and try again.' ) ?></p> 
     
    849717         
    850718        // Set up some options to avoid them autoloading (these ones get big) 
    851         add_option( 'ljapi_sync_item_times',        '', '', 'no' ); 
    852         add_option( 'ljapi_usermap',                '', '', 'no' ); 
    853         add_option( 'ljapi_comment_xml_files',      '', '', 'no' ); 
    854         add_option( 'ljapi_comment_cache_files',    '', '', 'no' ); 
    855         add_option( 'ljapi_comment_threaded_files', '', '', 'no' ); 
     719        add_option( 'ljapi_sync_item_times',  '', '', 'no' ); 
     720        add_option( 'ljapi_usermap',          '', '', 'no' ); 
     721        update_option( 'ljapi_comment_batch', 0 ); 
    856722         
    857723        echo '<h3>' . __( 'Importing Posts' ) . '</h3>'; 
     
    874740        } 
    875741         
    876         echo '<p>' . __( "Your posts have all been imported, but wait - there's more! Now we need to process &amp; import your comments." ) . '</p>'; 
     742        echo '<p>' . __( "Your posts have all been imported, but wait - there's more! Now we need to download &amp; import your comments." ) . '</p>'; 
    877743        echo $this->next_step( 2, __( 'Download my comments &raquo;' ) ); 
    878744        $this->auto_submit(); 
     
    885751        $this->username = get_option( 'ljapi_username' ); 
    886752        $this->password = get_option( 'ljapi_password' ); 
    887          
     753        $this->ixr = new IXR_Client( $this->ixr_url ); 
     754         
     755        echo '<div id="ljapi-status">'; 
    888756        echo '<h3>' . __( 'Downloading Comments' ) . '</h3>'; 
    889         echo '<p>' . __( 'Now we will download your comments so we can process and import them...' ) . '</p>'; 
     757        echo '<p>' . __( 'Now we will download your comments so we can import them (this could take a <strong>long</strong> time if you have lots of comments)...' ) . '</p>'; 
    890758        ob_flush(); flush(); 
    891759         
    892         $result = $this->download_comments(); 
     760        if ( !get_option( 'ljapi_usermap' ) ) { 
     761            // We haven't downloaded meta yet, so do that first 
     762            $result = $this->download_comment_meta(); 
     763            if ( is_wp_error( $result ) ) 
     764                return $result; 
     765        } 
     766 
     767        // Download a batch of actual comments 
     768        $result = $this->download_comment_bodies(); 
    893769        if ( is_wp_error( $result ) ) 
    894770            return $result; 
    895771 
    896         echo '<p>' . __( 'Your comments have all been downloaded to this server now, so we can process them and get them ready for importing.' ) . '</p>'; 
    897         echo $this->next_step( 3, __( 'Process my comment files &raquo;' ) ); 
    898         $this->auto_submit(); 
    899     } 
    900  
    901     // Parse XML into comment cache files    
     772        $maxid      = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1; 
     773        $highest_id = (int) get_option( 'ljapi_highest_comment_id' ); 
     774        if ( $maxid > $highest_id ) { 
     775        ?> 
     776            <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost"> 
     777            <p><strong><?php printf( __( 'Imported comment batch %d of <strong>approximately</strong> %d' ), get_option( 'ljapi_comment_batch' ), ( $maxid / 5000 ) ) ?></strong></p> 
     778            <?php wp_nonce_field( 'lj-api-import' ) ?> 
     779            <input type="hidden" name="step" id="step" value="2" /> 
     780            <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Import the next batch &raquo;' ) ) ?>" /> <span id="auto-message"></span></p> 
     781            </form> 
     782            <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> 
     783        <?php 
     784        } else { 
     785            echo '<p>' . __( 'Your comments have all been imported now, but we still need to rebuild your conversation threads.' ) . '</p>'; 
     786            echo $this->next_step( 3, __( 'Rebuild my comment threads &raquo;' ) ); 
     787            $this->auto_submit(); 
     788        } 
     789        echo '</div>'; 
     790    } 
     791     
     792    // Re-thread comments already in the DB 
    902793    function step3() { 
    903  
     794        global $wpdb; 
    904795        set_time_limit( 0 ); 
    905796        update_option( 'ljapi_step', 3 ); 
    906797         
    907         $this->usermap = get_option( 'ljapi_usermap' ); 
    908  
    909         echo '<div id="ljapi-status">'; 
    910         echo '<h3>' . __( 'Parsing Comments' ) . '</h3>'; 
    911         echo '<p>' . __( 'Time to clean up your comments and get them into a format WordPress understands...' ) . '</p>'; 
    912         ob_flush(); flush(); 
    913          
    914         $files = get_option( 'ljapi_comment_xml_files' ); 
    915         if ( count( $files ) ) { 
    916             $file = array_pop( $files ); 
    917          
    918             $result = $this->parse_comment_xml( $file ); 
    919             if ( is_wp_error( $result ) ) 
    920                 return $result; 
    921  
    922             update_option( 'ljapi_comment_xml_files', $files ); 
    923         } 
    924          
    925         if ( count( $files ) ) { 
    926             ?> 
    927                 <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost"> 
    928                 <p><strong><?php printf( __( 'Processed comment file %d of %d' ), ( get_option( 'ljapi_comment_xml_files_count' ) - count( $files ) ), get_option( 'ljapi_comment_xml_files_count' ) ) ?></strong></p> 
    929                 <?php wp_nonce_field( 'lj-api-import' ) ?> 
    930                 <input type="hidden" name="step" id="step" value="3" /> 
    931                 <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Process the next comment file &raquo;' ) ) ?>" /> <span id="auto-message"></span></p> 
    932                 </form> 
    933                 <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> 
    934             <?php 
    935         } else { 
    936             echo '<p>' . __( 'Yay, we finished processing all of your comment files! Now we need to re-build your conversation threads.' ) . '</p>'; 
    937             echo $this->next_step( 4, __( 'Thread my comments &raquo;' ) ); 
    938             $this->auto_submit(); 
    939         } 
    940         echo '</div>'; 
    941     } 
    942  
    943     // Thread comments within their cache files  
    944     function step4() { 
    945         set_time_limit( 0 ); 
    946         update_option( 'ljapi_step', 4 ); 
    947          
    948798        echo '<div id="ljapi-status">'; 
    949799        echo '<h3>' . __( 'Threading Comments' ) . '</h3>'; 
    950         echo '<p>' . __( 'Re-building your conversation threads ready for import...' ) . '</p>'; 
     800        echo '<p>' . __( 'We are now re-building the threading of your comments (this can also take a while if you have lots of comments)...' ) . '</p>'; 
    951801        ob_flush(); flush(); 
    952802         
    953         $files = get_option( 'ljapi_comment_cache_files' ); 
    954         if ( count( $files ) ) { 
    955             $file = array_pop( $files ); 
    956          
    957             $result = $this->thread_comments( $file ); 
    958             if ( is_wp_error( $result ) ) 
    959                 return $result; 
    960              
    961             update_option( 'ljapi_comment_cache_files', $files ); 
    962         } 
    963          
    964         if ( count( $files ) ) { 
    965             ?> 
    966                 <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost"> 
    967                 <p><strong><?php printf( __( 'Threaded cache file %d of %d' ), ( get_option( 'ljapi_comment_cache_files_count' ) - count( $files ) ), get_option( 'ljapi_comment_cache_files_count' ) ) ?></strong></p> 
    968                 <?php wp_nonce_field( 'lj-api-import' ) ?> 
    969                 <input type="hidden" name="step" id="step" value="4" /> 
    970                 <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Thread the next cache file &raquo;' ) ) ?>" /> <span id="auto-message"></span></p> 
    971                 </form> 
    972                 <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> 
    973             <?php 
    974         } else { 
    975             echo '<p>' . __( "Alrighty, your comments are all threaded. There's just one last step -- time to actually import them all now!" ) . '</p>'; 
    976             echo '<p>' . __( 'This last part in particular can take a really long time if you have a lot of comments. You might want to go and do something else while you wait.' ) . '</p>'; 
    977             echo $this->next_step( 5, __( 'Import my threaded comments into WordPress &raquo;' ) ); 
    978             $this->auto_submit(); 
    979         } 
    980         echo '</div>'; 
    981     } 
    982  
    983     // Import comments from cache files into WP 
    984     function step5() { 
    985         set_time_limit( 0 ); 
    986         update_option( 'ljapi_step', 5 ); 
    987          
    988          
    989         echo '<div id="ljapi-status">'; 
    990         echo '<h3>' . __( 'Importing Comments' ) . '</h3>'; 
    991         echo '<p>' . __( 'This is the big one -- we are now inserting your comment threads into WordPress...' ) . '</p>'; 
    992          
    993         $files = get_option( 'ljapi_comment_threaded_files' ); 
    994         echo '<p><strong>' . sprintf( __( 'Importing cache file %d of %d' ), ( get_option( 'ljapi_comment_threaded_files_count' ) - count( $files ) + 1 ), get_option( 'ljapi_comment_threaded_files_count' ) ) . '</strong></p>'; 
    995         ob_flush(); flush(); 
    996          
    997         if ( count( $files ) ) { 
    998             $file = array_pop( $files ); 
    999          
    1000             $result = $this->insert_comments( $file ); 
    1001             if ( is_wp_error( $result ) ) 
    1002                 return $result; 
    1003              
    1004             update_option( 'ljapi_comment_threaded_files', $files ); 
    1005         } 
    1006          
    1007         if ( count( $files ) ) { 
    1008             ?> 
    1009                 <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost"> 
    1010                 <?php wp_nonce_field( 'lj-api-import' ) ?> 
    1011                 <input type="hidden" name="step" id="step" value="5" /> 
    1012                 <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Import the next cache file &raquo;' ) ) ?>" /> <span id="auto-message"></span></p> 
    1013                 </form> 
    1014                 <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> 
    1015             <?php 
    1016         } else { 
    1017             // Clean up database and we're out 
    1018             $this->cleanup(); 
    1019             do_action( 'import_done', 'livejournal' ); 
    1020             echo '<h3>'; 
    1021             printf( __( 'All done. <a href="%s">Have fun!</a>' ), get_option( 'home' ) ); 
    1022             echo '</h3>'; 
    1023         } 
     803        // Only bother adding indexes if they have over 5000 comments (arbitrary number) 
     804        $imported_comments = $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->comments} WHERE comment_type = 'livejournal'" ); 
     805        $added_indices = false; 
     806        if ( 5000 < $imported_comments ) { 
     807            include_once(ABSPATH . 'wp-admin/includes/upgrade.php'); 
     808            $added_indices = true; 
     809            add_clean_index( $wpdb->comments, 'comment_type'  ); 
     810            add_clean_index( $wpdb->comments, 'comment_karma' ); 
     811            add_clean_index( $wpdb->comments, 'comment_agent' ); 
     812        } 
     813         
     814        // Get LJ comments, which haven't been threaded yet, 5000 at a time and thread them 
     815        while ( $comments = $wpdb->get_results( "SELECT comment_ID, comment_agent FROM {$wpdb->comments} WHERE comment_type = 'livejournal' AND comment_agent != '0' LIMIT 5000", OBJECT ) ) { 
     816            foreach ( $comments as $comment ) { 
     817                $wpdb->update( $wpdb->comments,  
     818                                array( 'comment_parent' => $this->get_wp_comment_ID( $comment->comment_agent ), 'comment_type' => 'livejournal-done' ),  
     819                                array( 'comment_ID' => $comment->comment_ID ) ); 
     820            } 
     821            wp_cache_flush(); 
     822            $wpdb->flush(); 
     823        } 
     824         
     825        // Revert the comments table back to normal and optimize it to reclaim space 
     826        if ( $added_indices ) { 
     827            drop_index( $wpdb->comments, 'comment_type'  ); 
     828            drop_index( $wpdb->comments, 'comment_karma' ); 
     829            drop_index( $wpdb->comments, 'comment_agent' ); 
     830            $wpdb->query( "OPTIMIZE TABLE {$wpdb->comments}" ); 
     831        } 
     832         
     833        // Clean up database and we're out 
     834        $this->cleanup(); 
     835        do_action( 'import_done', 'livejournal' ); 
     836        if ( $imported_comments > 1 ) 
     837            echo '<p>' . sprintf( __( "Successfully re-threaded %d comments." ), number_format( $imported_comments ) ) . '</p>'; 
     838        echo '<h3>'; 
     839        printf( __( 'All done. <a href="%s">Have fun!</a>' ), get_option( 'home' ) ); 
     840        echo '</h3>'; 
    1024841        echo '</div>'; 
    1025842    } 
     
    1035852         
    1036853        return $str; 
    1037     } 
    1038  
    1039     // Automatically submit the form with #id to continue the process 
    1040     // Hide any submit buttons to avoid people clicking them 
    1041     // Display a countdown in the element indicated by $msg for "Continuing in x" 
    1042     function auto_ajax( $id = 'ljapi-next-form', $msg = 'auto-message', $seconds = 5 ) { 
    1043         ?><script type="text/javascript"> 
    1044             next_counter = <?php echo $seconds ?>; 
    1045             jQuery(document).ready(function(){ 
    1046                 ljapi_msg(); 
    1047             }); 
    1048              
    1049             function ljapi_msg() { 
    1050                 str = '<?php _e( "Continuing in %d" ) ?>'; 
    1051                 jQuery( '#<?php echo $msg ?>' ).text( str.replace( /%d/, next_counter ) ); 
    1052                 if ( next_counter <= 0 ) { 
    1053                     if ( jQuery( '#<?php echo $id ?>' ).length ) { 
    1054                         jQuery( "#<?php echo $id ?> input[type='submit']" ).hide(); 
    1055                         jQuery.ajaxSetup({'timeout':3600000}); 
    1056                         str = '<?php _e( "Processing next file." ) ?> <img src="images/loading-publish.gif" alt="" id="processing" align="top" />'; 
    1057                         jQuery( '#<?php echo $msg ?>' ).html( str ); 
    1058                         jQuery('#ljapi-status').load(ajaxurl, {'action':'lj-importer', 
    1059                                                                 'step':jQuery('#step').val(), 
    1060                                                                 '_wpnonce':'<?php echo wp_create_nonce( 'lj-api-import' ) ?>', 
    1061                                                                 '_wp_http_referer':'<?php echo $_SERVER['REQUEST_URI'] ?>'}); 
    1062                         return; 
    1063                     } 
    1064                 } 
    1065                 next_counter = next_counter - 1; 
    1066                 setTimeout('ljapi_msg()', 1000); 
    1067             } 
    1068         </script><?php 
    1069854    } 
    1070855     
     
    1095880        </script><?php 
    1096881    } 
    1097  
    1098     // Remove all options used during import process 
     882     
     883    // Automatically submit the form with #id to continue the process 
     884    // Hide any submit buttons to avoid people clicking them 
     885    // Display a countdown in the element indicated by $msg for "Continuing in x" 
     886    function auto_ajax( $id = 'ljapi-next-form', $msg = 'auto-message', $seconds = 5 ) { 
     887        ?><script type="text/javascript"> 
     888            next_counter = <?php echo $seconds ?>; 
     889            jQuery(document).ready(function(){ 
     890                ljapi_msg(); 
     891            }); 
     892             
     893            function ljapi_msg() { 
     894                str = '<?php _e( "Continuing in %d" ) ?>'; 
     895                jQuery( '#<?php echo $msg ?>' ).text( str.replace( /%d/, next_counter ) ); 
     896                if ( next_counter <= 0 ) { 
     897                    if ( jQuery( '#<?php echo $id ?>' ).length ) { 
     898                        jQuery( "#<?php echo $id ?> input[type='submit']" ).hide(); 
     899                        jQuery.ajaxSetup({'timeout':3600000}); 
     900                        str = '<?php _e( "Processing next batch." ) ?> <img src="images/loading-publish.gif" alt="" id="processing" align="top" />'; 
     901                        jQuery( '#<?php echo $msg ?>' ).html( str ); 
     902                        jQuery('#ljapi-status').load(ajaxurl, {'action':'lj-importer', 
     903                                                                'step':jQuery('#step').val(), 
     904                                                                '_wpnonce':'<?php echo wp_create_nonce( 'lj-api-import' ) ?>', 
     905                                                                '_wp_http_referer':'<?php echo $_SERVER['REQUEST_URI'] ?>'}); 
     906                        return; 
     907                    } 
     908                } 
     909                next_counter = next_counter - 1; 
     910                setTimeout('ljapi_msg()', 1000); 
     911            } 
     912        </script><?php 
     913    } 
     914 
     915    // Remove all options used during import process and 
     916    // set wp_comments entries back to "normal" values 
    1099917    function cleanup() { 
     918        global $wpdb; 
     919         
    1100920        delete_option( 'ljapi_username' ); 
    1101921        delete_option( 'ljapi_password' ); 
     
    1112932        delete_option( 'ljapi_highest_id' ); 
    1113933        delete_option( 'ljapi_highest_comment_id' ); 
    1114         delete_option( 'ljapi_comment_xml_files' ); 
    1115         delete_option( 'ljapi_comment_xml_files_count' ); 
    1116         delete_option( 'ljapi_comment_cache_files' ); 
    1117         delete_option( 'ljapi_comment_cache_files_count' ); 
    1118         delete_option( 'ljapi_comment_threaded_files' ); 
    1119         delete_option( 'ljapi_comment_threaded_files_count' ); 
     934        delete_option( 'ljapi_comment_batch' ); 
    1120935        delete_option( 'ljapi_step' ); 
    1121     } 
    1122      
    1123     // Dump a string to a log file (appends to existing file) 
    1124     function log( $string, $name ) { 
    1125         return; // remove this to enable "debugging" output to files in /wp-content/ljimport 
    1126         $path = wp_upload_dir(); 
    1127         $path = $path['path']; 
    1128         if ( get_option( 'uploads_use_yearmonth_folders' ) ) 
    1129             $path = substr( $path, 0, -8 ); 
    1130  
    1131         if ( !is_dir( $path . '/ljimport' ) ) 
    1132             mkdir( $path . '/ljimport' ); 
    1133              
    1134         $fh = @fopen( $path . '/ljimport/' . $name, 'a' ); 
    1135         if ( $fh ) { 
    1136             if ( is_array( $string ) || is_object( $string ) ) 
    1137                 fwrite( $fh, var_export( $string, true ) . "\n\n" ); 
    1138             else 
    1139                 fwrite( $fh, $string . "\n\n" ); 
    1140             fclose( $fh ); 
    1141         } 
    1142     } 
    1143      
    1144     function write_file( $data, $name, $id, $mode = 'a' ) { 
    1145         if ( empty( $this->pointers[$id] ) ) 
    1146             $this->pointers[$id] = @fopen( $name, $mode ); 
    1147         if ( $this->pointers[$id] ) 
    1148             return fwrite( $this->pointers[$id], $data ); 
    1149         return false; 
    1150     } 
    1151      
    1152     function full_path( $basename ) { 
    1153         $uploads = wp_upload_dir(); 
    1154         return $uploads['path'] . '/' . $basename; 
    1155     } 
    1156      
    1157     function close_file_pointers() { 
    1158         foreach ( $this->pointers as $p ) 
    1159             @fclose( $p ); 
    1160     } 
    1161  
     936         
     937        $wpdb->update( $wpdb->comments,  
     938                        array( 'comment_karma' => 0, 'comment_agent' => 'WP LJ Importer', 'comment_type' => '' ),  
     939                        array( 'comment_type' => 'livejournal-done' ) ); 
     940        $wpdb->update( $wpdb->comments,  
     941                        array( 'comment_karma' => 0, 'comment_agent' => 'WP LJ Importer', 'comment_type' => '' ),  
     942                        array( 'comment_type' => 'livejournal' ) ); 
     943    } 
     944     
    1162945    function LJ_API_Import() { 
    1163946        $this->__construct(); 
Note: See TracChangeset for help on using the changeset viewer.