WordPress.org
Get WordPress

Make WordPress Core

Changeset 10538


Ignore:
Timestamp:
02/10/2009 07:34:52 PM (16 years ago)
Author:
azaozz
Message:

Completely New LiveJournal Importer update, props beaulebens, see #8999

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/wp-admin/import/livejournal.php

    r10501 r10538  
    11<?php
     2
    23/**
    34 * LiveJournal API Importer
    … …  
    910// XML-RPC library for communicating with LiveJournal API
    1011require_once( ABSPATH . WPINC . '/class-IXR.php' );
    11 
    12 // Snoopy for getting comments (with cookies)
    13 require_once( ABSPATH . WPINC . '/class-snoopy.php' );
    1412
    1513/**
    … …  
    2725    var $username;
    2826    var $password;
    29     var $snoop;
    3027    var $comment_meta;
    3128    var $comments;
    3229    var $usermap;
    3330    var $postmap;
     31    var $commentmap;
    3432    var $pointers = array();
    3533   
    … …  
    189187                <input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Continue previous import' ) ) ?>" />
    190188            </p>
    191             <p><a href="<?php echo $_SERVER['PHP_SELF'] . '?import=livejournal&amp;step=-1&amp;_wpnonce=' . wp_create_nonce( 'lj-api-import' ) . '&amp;_wp_http_referer=' . attribute_escape( $_SERVER['REQUEST_URI'] ) ?>"><?php _e( 'Cancel &amp; start a new import' ) ?></a></p>
     189            <p class="submitbox"><a href="<?php echo $_SERVER['PHP_SELF'] . '?import=livejournal&amp;step=-1&amp;_wpnonce=' . wp_create_nonce( 'lj-api-import' ) . '&amp;_wp_http_referer=' . attribute_escape( $_SERVER['REQUEST_URI'] ) ?>" class="deletion submitdelete"><?php _e( 'Cancel &amp; start a new import' ) ?></a></p>
    192190            <p>
    193191        <?php else : ?>
    … …  
    229227       
    230228            <p><?php _e( '<strong>NOTE:</strong> If the import process is interrupted for <em>any</em> reason, come back to this page and it will continue from where it stopped automatically.' ) ?></p>
     229
     230            <noscript>
     231                <p><?php _e( '<strong>NOTE:</strong> You appear to have JavaScript disabled, so you will need to manually click through each step of this importer. If you enable JavaScript, it will step through automatically.' ) ?></p>
     232            </noscript>
    231233        <?php endif; ?>
    232234        </form>
    … …  
    249251            $lastsync = date( 'Y-m-d H:i:s', strtotime( get_option( 'ljapi_lastsync' ) ) );
    250252            $synclist = $this->lj_ixr( 'syncitems', array( 'ver' => 1, 'lastsync' => $lastsync ) );
    251             $this->log( $synclist, 'ljimport-items-' . $total . '.txt' );
    252253           
    253254            // Keep track of if we've downloaded everything
    … …  
    262263                }
    263264            }
    264 
    265             update_option( 'ljapi_sync_item_times', $sync_item_times );
    266             update_option( 'ljapi_total', $total );
    267             update_option( 'ljapi_count', $count );
    268             update_option( 'ljapi_lastsync', $lastsync );
    269265        } while ( $total > $count );
    270266        // endwhile - all post meta is cached locally
    271         $this->log( $sync_item_times, 'ljimport-post-mod-times.txt' );
     267       
     268        update_option( 'ljapi_sync_item_times', $sync_item_times );
     269        update_option( 'ljapi_total', $total );
     270        update_option( 'ljapi_count', $count );
     271        update_option( 'ljapi_lastsync', $lastsync );
     272
     273        echo '<p>' . __( 'Post metadata has been downloaded, proceeding with posts...' ) . '</p>';
    272274       
    273275        echo '<ol>';
    … …  
    286288                                                            'lineendings' => 'pc',
    287289                                                            'lastsync' => $lastsync ) );
    288             $this->log( $itemlist, 'ljimport-posts-' . $imported_count . '.txt' );
    289290            if ( is_wp_error( $itemlist ) )
    290291                return $itemlist;
     292               
    291293            if ( $num = count( $itemlist['events'] ) ) {
    292294                foreach ( $itemlist['events'] as $event ) {
    … …  
    313315       
    314316        $user = wp_get_current_user();
    315         $post_author   = $user->ID;
    316         $post_status   = ( 'private' == trim( $post['security'] ) ) ? 'private' : 'publish'; // Only me
    317         $post_password = ( 'usemask' == trim( $post['security'] ) ) ? $this->protected_password : ''; // "Friends" via password
     317        $post_author      = $user->ID;
     318        $post['security'] = !empty( $post['security'] ) ? $post['security'] : '';
     319        $post_status      = ( 'private' == trim( $post['security'] ) ) ? 'private' : 'publish'; // Only me
     320        $post_password    = ( 'usemask' == trim( $post['security'] ) ) ? $this->protected_password : ''; // "Friends" via password
    318321
    319322        // For some reason, LJ sometimes sends a date as "2004-04-1408:38:00" (no space btwn date/time)
    … …  
    323326       
    324327        // Cleaning up and linking the title
    325         $post_title = trim( $post['subject'] );
     328        $post_title = isset( $post['subject'] ) ? trim( $post['subject'] ) : '';
    326329        $post_title = $this->translate_lj_user( $post_title ); // Translate it, but then we'll strip the link
    327330        $post_title = strip_tags( $post_title ); // Can't have tags in the title in WP
    … …  
    366369            $postdata['post_ID']   = $post_id;
    367370            $postdata['lj_itemid'] = $post['itemid'];
    368             $this->log( $postdata, 'ljimport-post-' . $post_id . '.txt' );
    369371           
    370372            // Handle all the metadata for this post
    … …  
    405407    }
    406408   
    407     // Loops through and gets comment meta and content from LJ in batches
    408     // Writes raw XML files to disk for later processing
    409     function download_comments() {
     409    // Set up a session (authenticate) with LJ
     410    function getsession() {
    410411        // Get a session via XMLRPC
    411412        $cookie = $this->lj_ixr( 'sessiongenerate', array( 'ver' => 1, 'expiration' => 'short' ) );
    412        
    413         // Comment Meta
     413        return new WP_Http_Cookie( array( 'name' => 'ljsession', 'value' => $cookie['ljsession'] ) );
     414    }
     415   
     416    // Loops through and gets comment meta from LJ in batches
     417    function download_comment_meta() {
     418        $cookie = $this->getsession();
    414419       
    415420        // Load previous state (if any)
    416421        $this->usermap = (array) get_option( 'ljapi_usermap' );
    417         $maxid         = (int) get_option( 'ljapi_maxid' ) || 1;
    418         $highest_id    = (int) get_option( 'ljapi_highest_id' );
    419 
    420         // Snoopy is required to handle the cookie
    421         $this->snoop = new Snoopy();
    422         $this->snoop->cookies = $cookie;
     422        $maxid         = get_option( 'ljapi_maxid' ) ? get_option( 'ljapi_maxid' ) : 1;
     423        $highest_id    = get_option( 'ljapi_highest_id' ) ? get_option( 'ljapi_highest_id' ) : 0;
    423424       
    424425        // We need to loop over the metadata request until we have it all
    425426        while ( $maxid > $highest_id ) {
    426427            // Now get the meta listing
    427             if ( !$this->snoop->fetch( $this->comments_url . '?get=comment_meta&startid=' . ( $highest_id + 1 ) ) )
    428                 return new WP_Error( 'Snoopy', __( 'Failed to retrieve comment meta information from LiveJournal. Please try again soon.' ) );
    429 
    430             // Snoopy doesn't provide an accessor for results...
    431             $results = $this->snoop->results;
     428            $results = wp_remote_get( $this->comments_url . '?get=comment_meta&startid=' . ( $highest_id + 1 ),
     429                                        array( 'cookies' => array( $cookie ), 'timeout' => 20 ) );
     430            if ( is_wp_error( $results ) )
     431                return new WP_Error( 'comment_meta', __( 'Failed to retrieve comment meta information from LiveJournal. Please try again soon.' ) );
     432           
     433            $results = wp_remote_retrieve_body( $results );
    432434           
    433435            // Get the maxid so we know if we have them all yet
    434436            preg_match( '|<maxid>(\d+)</maxid>|', $results, $matches );
     437            if ( 0 == $matches[1] ) {
     438                // No comment meta = no comments for this journal
     439                echo '<p>' . __( 'You have no comments to import!' ) . '</p>';
     440                update_option( 'ljapi_highest_id', 1 );
     441                update_option( 'ljapi_highest_comment_id', 1 );
     442                return false; // Bail out of comment importing entirely
     443            }
    435444            $maxid = !empty( $matches[1] ) ? $matches[1] : $maxid;
    436445           
    … …  
    447456                $this->usermap[$userid] = $matches[2][$count]; // need this in memory for translating ids => names
    448457               
    449             update_option( 'ljapi_usermap',    $this->usermap );
    450             update_option( 'ljapi_maxid',      $maxid );
    451             update_option( 'ljapi_highest_id', $highest_id );
     458            wp_cache_flush();
    452459        }
    453460        // endwhile - should have seen all comment meta at this point
    454        
    455        
    456         // Download Comment XML
     461
     462        update_option( 'ljapi_usermap',    $this->usermap );
     463        update_option( 'ljapi_maxid',      $maxid );
     464        update_option( 'ljapi_highest_id', $highest_id );
     465       
     466        echo '<p>' . __( ' Comment metadata downloaded successfully, proceeding with posts...' ) . '</p>';
     467       
     468        return true;
     469    }
     470   
     471    // Downloads actual comment bodies from LJ
     472    // Inserts them all directly to the DB, with additional info stored in "spare" fields
     473    function download_comment_bodies() {
     474        global $wpdb;
     475        $cookie = $this->getsession();
    457476       
    458477        // Load previous state (if any)
    459         $highest_id          = (int) get_option( 'ljapi_highest_comment_id' );
    460         $comment_xml_files   = get_option( 'ljapi_comment_xml_files' );
    461         if ( !is_array( $comment_xml_files ) ) {
    462             update_option( 'ljapi_comment_xml_files', array() );
    463             $comment_xml_files = array();
    464         }
    465        
    466         echo '<ol>';
    467        
    468         // And now request the actual comments, and keep going until we have them all
    469         while ( $maxid > $highest_id ) {
     478        $this->usermap = (array) get_option( 'ljapi_usermap' );
     479        $maxid         = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1;
     480        $highest_id    = (int) get_option( 'ljapi_highest_comment_id' );
     481        $loop = 0;
     482        while ( $maxid > $highest_id && $loop < 5 ) { // We do 5 loops per call to avoid memory limits
     483            $loop++;
     484           
    470485            // Get a batch of comments, using the highest_id we've already got as a starting point
    471             if ( !$this->snoop->fetch( $this->comments_url . '?get=comment_body&startid=' . ( $highest_id + 1 ) ) )
    472                 return new WP_Error( 'Snoopy', __( 'Failed to retrieve comment bodies from LiveJournal. Please try again soon.' ) );
    473            
    474             // Get the highest post ID in this batch (required for loop control)
    475             $results = $this->snoop->results;
    476             preg_match_all( '|<comment id=\'(\d+)\'|i', $results, $comments );
    477             for ( $r = 0; $r < count( $comments[1] ); $r++ ) {
    478                 if ( $comments[1][$r] > $highest_id )
    479                     $highest_id = $comments[1][$r];
    480             }
    481            
    482             // $this->snoop-results is where the actual response is stored
    483             $this->log( $this->snoop->results, 'ljimport-comment-bodies-' . $highest_id . '.txt' );
    484            
    485             // Store in uploads dir. Can't use *.xml because it's not allowed
    486             $results = wp_upload_bits( 'raw-comments-' . $highest_id . '.txt', null, $results );
    487             if ( !empty( $results['error'] ) )
    488                 return new WP_Error( 'xml', $results['error'] );
    489             $comment_xml_files[] = $results['file'];
    490            
    491             echo '<li>' . sprintf( __( 'Downloaded <strong>%s</strong>' ), basename( $results['file'] ) ) . '</li>';
    492             ob_flush(); flush();
    493            
    494             $comment_xml_files = array_unique( $comment_xml_files );
    495             update_option( 'ljapi_comment_xml_files', $comment_xml_files );
    496             update_option( 'ljapi_comment_xml_files_count', count( $comment_xml_files ) );
     486            $results = wp_remote_get( $this->comments_url . '?get=comment_body&startid=' . ( $highest_id + 1 ),
     487                                        array( 'cookies' => array( $cookie ), 'timeout' => 20 ) );
     488            if ( is_wp_error( $results ) )
     489                return new WP_Error( 'comment_bodies', __( 'Failed to retrieve comment bodies from LiveJournal. Please try again soon.' ) );
     490           
     491            $results = wp_remote_retrieve_body( $results );
     492           
     493            // Parse out each comment and insert directly
     494            preg_match_all( '|<comment id=\'(\d+)\'.*</comment>|iUs', $results, $matches );
     495            for ( $c = 0; $c < count( $matches[0] ); $c++ ) {
     496                // Keep track of highest id seen
     497                if ( $matches[1][$c] > $highest_id ) {
     498                    $highest_id = $matches[1][$c];
     499                    update_option( 'ljapi_highest_comment_id', $highest_id );
     500                }
     501                   
     502                $comment = $matches[0][$c];
     503
     504                // Filter out any captured, deleted comments (nothing useful to import)
     505                $comment = preg_replace( '|<comment id=\'\d+\' jitemid=\'\d+\' posterid=\'\d+\' state=\'D\'[^/]*/>|is', '', $comment );
     506
     507                // Parse this comment into an array and insert
     508                $comment = $this->parse_comment( $comment );
     509                $id = wp_insert_comment( $comment );
     510
     511                // Clear cache
     512                clean_comment_cache( $id );
     513            }
     514           
     515            // Clear cache to preseve memory
     516            wp_cache_flush();
    497517        }
    498518        // endwhile - all comments downloaded and ready for bulk processing
    499519       
    500         echo '</ol>';
     520        // Counter just used to show progress to user
     521        update_option( 'ljapi_comment_batch', ( (int) get_option( 'ljapi_comment_batch' ) + 1 ) );
    501522       
    502523        return true;
    503524    }
    504525   
    505     function parse_comment_xml( $xml_file ) {
    506         if ( !is_file( $xml_file ) || !is_readable( $xml_file ) )
    507             return new WP_Error( 'file', sprintf( __( 'Could not access comment XML file: %s'), $filename ) );
    508            
    509         // Get content from file
    510         $xml = @file_get_contents( $xml_file );
    511 
    512         $cache_files = get_option( 'ljapi_comment_cache_files' );
    513         if ( !is_array( $cache_files ) )
    514             $cache_files = array();
    515        
    516         // Parse XML into comments
    517         preg_match_all( '|<comment id.*</comment>|iUs', $xml, $matches );
    518         unset( $xml );
    519         for ( $c = 0; $c < count( $matches[0] ); $c++ ) {
    520             $comment = $matches[0][$c];
    521            
    522             // Filter out any captured, deleted comments (nothing useful to import)
    523             $comment = preg_replace( '|<comment id=\'\d+\' jitemid=\'\d+\' posterid=\'\d+\' state=\'D\'[^/]*/>|is', '', $comment );
    524            
    525             // Parse this comment into an array
    526             $comment = $this->parse_comment( $comment );
    527             if ( empty( $comment['comment_post_ID'] ) )
    528                 continue;
    529            
    530             // Add this comment to the appropriate cache file
    531             $filename = $this->full_path( 'ljimport-comments-' . $comment['comment_post_ID'] . '.php' );
    532             if ( $this->write_file( '<?php $comments[] = ' . var_export( $comment, true ) . '; ?>' . "\n",
    533                                 $filename,
    534                                 $comment['comment_post_ID'],
    535                                 'a' ) )
    536             {
    537                 // Keep track of files used
    538                 $cache_files[] = $filename;
    539             }
    540         }
    541        
    542         // Update list of files in the DB
    543         sort( $cache_files );
    544         $cache_files = array_unique( $cache_files );
    545         update_option( 'ljapi_comment_cache_files', $cache_files );
    546         update_option( 'ljapi_comment_cache_files_count', count( $cache_files ) );
    547         $this->close_file_pointers();
    548        
    549         // Don't need this XML file anymore
    550         unlink( $xml_file );
    551        
    552         return true;
    553     }
    554    
     526    // Takes a block of XML and parses out all the elements of the comment
    555527    function parse_comment( $comment ) {
    556528        global $wpdb;
    … …  
    563535        $lj_comment_post_ID = $matches[1];
    564536        preg_match( '| posterid=\'(\d+)\'|i', $attribs[1], $matches );
    565         $comment_author_ID = $matches[1];
    566         preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches );
    567         $lj_comment_parent = $matches[1];
    568         preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches );
    569         $lj_comment_state = !empty( $matches[1] ) ? $matches[1] : 'A';
     537        $comment_author_ID = isset( $matches[1] ) ? $matches[1] : 0;
     538        preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches ); // optional
     539        $lj_comment_parent = isset( $matches[1] ) ? $matches[1] : 0;
     540        preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches ); // optional
     541        $lj_comment_state = isset( $matches[1] ) ? $matches[1] : 'A';
    570542       
    571543        // Clean up "subject" - this will become the first line of the comment in WP
    572544        preg_match( '|<subject>(.*)</subject>|is', $comment, $matches );
    573         $comment_subject = $wpdb->escape( trim( $matches[1] ) );
    574         if ( 'Re:' == $comment_subject )
    575             $comment_subject = '';
     545        if ( isset( $matches[1] ) ) {
     546            $comment_subject = $wpdb->escape( trim( $matches[1] ) );
     547            if ( 'Re:' == $comment_subject )
     548                $comment_subject = '';
     549        }
    576550       
    577551        // Get the body and HTMLize it
    … …  
    591565       
    592566        // Grab IP if available
    593         preg_match( '|<property name=\'poster_ip\'>(.*)</property>|i', $comment, $matches );
    594         $comment_author_IP = $matches[1];
     567        preg_match( '|<property name=\'poster_ip\'>(.*)</property>|i', $comment, $matches ); // optional
     568        $comment_author_IP = isset( $matches[1] ) ? $matches[1] : '';
    595569       
    596570        // Try to get something useful for the comment author, especially if it was "my" comment
    597         $author = ( substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' || empty( $comment_author_ID ) ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID];
     571        $author = ( empty( $comment_author_ID ) || empty( $this->usermap[$comment_author_ID] ) || substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID];
    598572        if ( get_option( 'ljapi_username' ) == $author ) {
    599573            $user    = wp_get_current_user();
    … …  
    614588                        'comment_author' => $author,
    615589                        'comment_author_url' => $url,
     590                        'comment_author_email' => '',
    616591                        'comment_content' => $comment_content,
    617592                        'comment_date' => $comment_date,
    618593                        'comment_author_IP' => ( !empty( $comment_author_IP ) ? $comment_author_IP : '' ),
    619594                        'comment_approved' => ( in_array( $lj_comment_state, array( 'A', 'F' ) ) ? 1 : 0 ),
    620                         'comment_agent' => 'WP LJ Importer',
    621                         'user_id' => $user_id
    622                         );
     595                        'comment_karma' => $lj_comment_ID, // Need this and next value until rethreading is done
     596                        'comment_agent' => $lj_comment_parent,
     597                        'comment_type' => 'livejournal',  // Custom type, so we can find it later for processing
     598                        'user_ID' => $user_id
     599                    );
    623600    }
    624601   
    … …  
    628605        global $wpdb;
    629606        if ( empty( $this->postmap[$post] ) )
    630             $this->postmap[$post] = $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) );
     607            $this->postmap[$post] = (int) $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) );
    631608        return $this->postmap[$post];
    632609    }
    633610   
    634     // Re-build the threading within a single cache file
    635     function thread_comments( $filename ) {
    636         if ( !is_file( $filename ) || !is_readable( $filename ) )
    637             return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) );
    638        
    639         $comments = array();
    640         @include( $filename );
    641         $this->comments = $comments;
    642         unset( $comments );
    643         if ( !is_array( $this->comments ) )
    644             $this->comments = array();
    645            
    646         $count = count( $this->comments );
    647         for ( $c = 0; $c < $count; $c++ ) {
    648             // Skip anything that's not "top-level" for now
    649             if ( 0 != $this->comments[$c]['lj_comment_parent'] )
    650                 continue;
    651             $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] );
    652         }
    653        
    654         // Remove anything that's not supposed to be at top level
    655         $top_comments = array();
    656         for ( $c = 0; $c < $count; $c++ ) {
    657             if ( 0 == $this->comments[$c]['lj_comment_parent'] ) {
    658                 $top_comments[] = $this->comments[$c];
    659             }
    660         }
    661        
    662         // Write back to file
    663         @unlink( $filename );
    664         $this->write_file( '<?php $comments = ' . var_export( $top_comments, true ) . '; ?>', $filename, $count, 'w' );
    665         unset( $top_comments );
    666         $this->close_file_pointers();
    667        
    668         // Reference this file as being threaded
    669         $files = get_option( 'ljapi_comment_threaded_files' );
    670         $files[] = $filename;
    671         array_unique( $files );
    672         update_option( 'ljapi_comment_threaded_files', $files );
    673         update_option( 'ljapi_comment_threaded_files_count', count( $files ) );
    674        
    675         return true;
    676     }
    677    
    678     function get_child_comments( $id ) {
    679         $children = array();
    680         $count = count( $this->comments );
    681         for ( $c = 0; $c < $count; $c++ ) {
    682             // This comment is a child of the $id
    683             if ( $id == $this->comments[$c]['lj_comment_parent'] ) {
    684                 $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] );
    685                 $children[] = $this->comments[$c];
    686             }
    687         }
    688         return $children;
    689     }
    690    
    691     // Inserts the contents of each cache file (should be threaded already)
    692     function insert_comments( $filename ) {
    693         echo '<ol>';
    694 
    695         if ( !is_file( $filename ) || !is_readable( $filename ) )
    696             return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) );
    697        
    698         $comments = array();
    699         @include( $filename );
    700         $this->comments = $comments;
    701         unset( $comments );
    702         if ( !is_array( $this->comments ) )
    703             $this->comments = array();
    704            
    705         $count = count( $this->comments );
    706         for ( $c = 0; $c < $count; $c++ ) {
    707             $comment =& $this->comments[$c];
    708             echo '<li>';
    709             printf( __( 'Imported comment from <strong>%s</strong> on %s' ), $comment['comment_author'], $comment['comment_date'] );
    710 
    711             $id = wp_insert_comment( $comment );
    712             $comment['comment_ID'] = $id;
    713             if ( count( $comment['children'] ) ) {
    714                 _e( ' and replies:' );
    715                 $this->insert_child_comments( $comment['children'], $id );
    716             }
    717            
    718             echo '</li>';
    719         }
    720        
    721         // Remove the file now that we're done with it
    722         @unlink( $filename );
    723 
    724         echo '</ol>';
    725        
    726         return true;
    727     }
    728    
    729     function insert_child_comments( &$comments, $parent ) {
    730         echo '<ol>';
    731         $count = count( $comments );
    732         for ( $c = 0; $c < $count; $c++ ) {
    733             $comment =& $comments[$c];
    734             $comment['comment_parent'] = $parent;
    735             echo '<li>';
    736             printf( __( 'Imported reply from <strong>%s</strong> on %s' ), $comment['comment_author'], $comment['comment_date'] );
    737 
    738             $id = wp_insert_comment( $comment );
    739             $comment['comment_ID'] = $id;
    740             if ( count( $comment['children'] ) ) {
    741                 _e( ' and replies:' );
    742                 $this->insert_child_comments( $comment['children'], $id );
    743             }
    744            
    745             echo '</li>';
    746         }
    747         echo '</ol>';
     611    // Gets the comment_ID that a LJ comment has been saved as within WP
     612    function get_wp_comment_ID( $comment ) {
     613        global $wpdb;
     614        if ( empty( $this->commentmap[$comment] ) )
     615            $this->commentmap[$comment] = $wpdb->get_var( $wpdb->prepare( "SELECT comment_ID FROM $wpdb->comments WHERE comment_karma = %d", $comment ) );
     616        return $this->commentmap[$comment];
    748617    }
    749618           
    … …  
    768637            return $this->ixr->getResponse();
    769638        } else {
    770             $this->log( $this->ixr->message, 'ljimport-error-' . $method . '.txt' );
    771639            return new WP_Error( 'IXR', __( 'XML-RPC Request Failed - ' ) . $this->ixr->getErrorCode() . ': ' . $this->ixr->getErrorMessage() );
    772640        }
    … …  
    789657                break;
    790658            case 1 :
    791             case 2 :
    792659                $this->ixr = new IXR_Client( $this->ixr_url );
    793660                // Intentional no break
     661            case 2 :
    794662            case 3 :
    795             case 4 :
    796             case 5 :
    797663                check_admin_referer( 'lj-api-import' );
    798664                $result = $this->{ 'step' . $step }();
    … …  
    838704        if ( is_wp_error( $login ) ) {
    839705            if ( 100 == $this->ixr->getErrorCode() || 101 == $this->ixr->getErrorCode() ) {
     706                delete_option( 'ljapi_username' );
     707                delete_option( 'ljapi_password' );
    840708                ?>
    841709                <p><?php _e( 'Logging in to LiveJournal failed. Check your username and password and try again.' ) ?></p>
    … …  
    849717       
    850718        // Set up some options to avoid them autoloading (these ones get big)
    851         add_option( 'ljapi_sync_item_times',        '', '', 'no' );
    852         add_option( 'ljapi_usermap',                '', '', 'no' );
    853         add_option( 'ljapi_comment_xml_files',      '', '', 'no' );
    854         add_option( 'ljapi_comment_cache_files',    '', '', 'no' );
    855         add_option( 'ljapi_comment_threaded_files', '', '', 'no' );
     719        add_option( 'ljapi_sync_item_times',  '', '', 'no' );
     720        add_option( 'ljapi_usermap',          '', '', 'no' );
     721        update_option( 'ljapi_comment_batch', 0 );
    856722       
    857723        echo '<h3>' . __( 'Importing Posts' ) . '</h3>';
    … …  
    874740        }
    875741       
    876         echo '<p>' . __( "Your posts have all been imported, but wait - there's more! Now we need to process &amp; import your comments." ) . '</p>';
     742        echo '<p>' . __( "Your posts have all been imported, but wait - there's more! Now we need to download &amp; import your comments." ) . '</p>';
    877743        echo $this->next_step( 2, __( 'Download my comments &raquo;' ) );
    878744        $this->auto_submit();
    … …  
    885751        $this->username = get_option( 'ljapi_username' );
    886752        $this->password = get_option( 'ljapi_password' );
    887        
     753        $this->ixr = new IXR_Client( $this->ixr_url );
     754       
     755        echo '<div id="ljapi-status">';
    888756        echo '<h3>' . __( 'Downloading Comments' ) . '</h3>';
    889         echo '<p>' . __( 'Now we will download your comments so we can process and import them...' ) . '</p>';
     757        echo '<p>' . __( 'Now we will download your comments so we can import them (this could take a <strong>long</strong> time if you have lots of comments)...' ) . '</p>';
    890758        ob_flush(); flush();
    891759       
    892         $result = $this->download_comments();
     760        if ( !get_option( 'ljapi_usermap' ) ) {
     761            // We haven't downloaded meta yet, so do that first
     762            $result = $this->download_comment_meta();
     763            if ( is_wp_error( $result ) )
     764                return $result;
     765        }
     766
     767        // Download a batch of actual comments
     768        $result = $this->download_comment_bodies();
    893769        if ( is_wp_error( $result ) )
    894770            return $result;
    895771
    896         echo '<p>' . __( 'Your comments have all been downloaded to this server now, so we can process them and get them ready for importing.' ) . '</p>';
    897         echo $this->next_step( 3, __( 'Process my comment files &raquo;' ) );
    898         $this->auto_submit();
    899     }
    900 
    901     // Parse XML into comment cache files   
     772        $maxid      = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1;
     773        $highest_id = (int) get_option( 'ljapi_highest_comment_id' );
     774        if ( $maxid > $highest_id ) {
     775        ?>
     776            <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost">
     777            <p><strong><?php printf( __( 'Imported comment batch %d of <strong>approximately</strong> %d' ), get_option( 'ljapi_comment_batch' ), ( $maxid / 5000 ) ) ?></strong></p>
     778            <?php wp_nonce_field( 'lj-api-import' ) ?>
     779            <input type="hidden" name="step" id="step" value="2" />
     780            <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Import the next batch &raquo;' ) ) ?>" /> <span id="auto-message"></span></p>
     781            </form>
     782            <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
     783        <?php
     784        } else {
     785            echo '<p>' . __( 'Your comments have all been imported now, but we still need to rebuild your conversation threads.' ) . '</p>';
     786            echo $this->next_step( 3, __( 'Rebuild my comment threads &raquo;' ) );
     787            $this->auto_submit();
     788        }
     789        echo '</div>';
     790    }
     791   
     792    // Re-thread comments already in the DB
    902793    function step3() {
    903 
     794        global $wpdb;
    904795        set_time_limit( 0 );
    905796        update_option( 'ljapi_step', 3 );
    906797       
    907         $this->usermap = get_option( 'ljapi_usermap' );
    908 
    909         echo '<div id="ljapi-status">';
    910         echo '<h3>' . __( 'Parsing Comments' ) . '</h3>';
    911         echo '<p>' . __( 'Time to clean up your comments and get them into a format WordPress understands...' ) . '</p>';
    912         ob_flush(); flush();
    913        
    914         $files = get_option( 'ljapi_comment_xml_files' );
    915         if ( count( $files ) ) {
    916             $file = array_pop( $files );
    917        
    918             $result = $this->parse_comment_xml( $file );
    919             if ( is_wp_error( $result ) )
    920                 return $result;
    921 
    922             update_option( 'ljapi_comment_xml_files', $files );
    923         }
    924        
    925         if ( count( $files ) ) {
    926             ?>
    927                 <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost">
    928                 <p><strong><?php printf( __( 'Processed comment file %d of %d' ), ( get_option( 'ljapi_comment_xml_files_count' ) - count( $files ) ), get_option( 'ljapi_comment_xml_files_count' ) ) ?></strong></p>
    929                 <?php wp_nonce_field( 'lj-api-import' ) ?>
    930                 <input type="hidden" name="step" id="step" value="3" />
    931                 <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Process the next comment file &raquo;' ) ) ?>" /> <span id="auto-message"></span></p>
    932                 </form>
    933                 <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
    934             <?php
    935         } else {
    936             echo '<p>' . __( 'Yay, we finished processing all of your comment files! Now we need to re-build your conversation threads.' ) . '</p>';
    937             echo $this->next_step( 4, __( 'Thread my comments &raquo;' ) );
    938             $this->auto_submit();
    939         }
    940         echo '</div>';
    941     }
    942 
    943     // Thread comments within their cache files
    944     function step4() {
    945         set_time_limit( 0 );
    946         update_option( 'ljapi_step', 4 );
    947        
    948798        echo '<div id="ljapi-status">';
    949799        echo '<h3>' . __( 'Threading Comments' ) . '</h3>';
    950         echo '<p>' . __( 'Re-building your conversation threads ready for import...' ) . '</p>';
     800        echo '<p>' . __( 'We are now re-building the threading of your comments (this can also take a while if you have lots of comments)...' ) . '</p>';
    951801        ob_flush(); flush();
    952802       
    953         $files = get_option( 'ljapi_comment_cache_files' );
    954         if ( count( $files ) ) {
    955             $file = array_pop( $files );
    956        
    957             $result = $this->thread_comments( $file );
    958             if ( is_wp_error( $result ) )
    959                 return $result;
    960            
    961             update_option( 'ljapi_comment_cache_files', $files );
    962         }
    963        
    964         if ( count( $files ) ) {
    965             ?>
    966                 <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost">
    967                 <p><strong><?php printf( __( 'Threaded cache file %d of %d' ), ( get_option( 'ljapi_comment_cache_files_count' ) - count( $files ) ), get_option( 'ljapi_comment_cache_files_count' ) ) ?></strong></p>
    968                 <?php wp_nonce_field( 'lj-api-import' ) ?>
    969                 <input type="hidden" name="step" id="step" value="4" />
    970                 <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Thread the next cache file &raquo;' ) ) ?>" /> <span id="auto-message"></span></p>
    971                 </form>
    972                 <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
    973             <?php
    974         } else {
    975             echo '<p>' . __( "Alrighty, your comments are all threaded. There's just one last step -- time to actually import them all now!" ) . '</p>';
    976             echo '<p>' . __( 'This last part in particular can take a really long time if you have a lot of comments. You might want to go and do something else while you wait.' ) . '</p>';
    977             echo $this->next_step( 5, __( 'Import my threaded comments into WordPress &raquo;' ) );
    978             $this->auto_submit();
    979         }
    980         echo '</div>';
    981     }
    982 
    983     // Import comments from cache files into WP
    984     function step5() {
    985         set_time_limit( 0 );
    986         update_option( 'ljapi_step', 5 );
    987        
    988        
    989         echo '<div id="ljapi-status">';
    990         echo '<h3>' . __( 'Importing Comments' ) . '</h3>';
    991         echo '<p>' . __( 'This is the big one -- we are now inserting your comment threads into WordPress...' ) . '</p>';
    992        
    993         $files = get_option( 'ljapi_comment_threaded_files' );
    994         echo '<p><strong>' . sprintf( __( 'Importing cache file %d of %d' ), ( get_option( 'ljapi_comment_threaded_files_count' ) - count( $files ) + 1 ), get_option( 'ljapi_comment_threaded_files_count' ) ) . '</strong></p>';
    995         ob_flush(); flush();
    996        
    997         if ( count( $files ) ) {
    998             $file = array_pop( $files );
    999        
    1000             $result = $this->insert_comments( $file );
    1001             if ( is_wp_error( $result ) )
    1002                 return $result;
    1003            
    1004             update_option( 'ljapi_comment_threaded_files', $files );
    1005         }
    1006        
    1007         if ( count( $files ) ) {
    1008             ?>
    1009                 <form action="admin.php?import=livejournal" method="post" id="ljapi-auto-repost">
    1010                 <?php wp_nonce_field( 'lj-api-import' ) ?>
    1011                 <input type="hidden" name="step" id="step" value="5" />
    1012                 <p><input type="submit" class="button-primary" value="<?php echo attribute_escape( __( 'Import the next cache file &raquo;' ) ) ?>" /> <span id="auto-message"></span></p>
    1013                 </form>
    1014                 <?php $this->auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?>
    1015             <?php
    1016         } else {
    1017             // Clean up database and we're out
    1018             $this->cleanup();
    1019             do_action( 'import_done', 'livejournal' );
    1020             echo '<h3>';
    1021             printf( __( 'All done. <a href="%s">Have fun!</a>' ), get_option( 'home' ) );
    1022             echo '</h3>';
    1023         }
     803        // Only bother adding indexes if they have over 5000 comments (arbitrary number)
     804        $imported_comments = $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->comments} WHERE comment_type = 'livejournal'" );
     805        $added_indices = false;
     806        if ( 5000 < $imported_comments ) {
     807            include_once(ABSPATH . 'wp-admin/includes/upgrade.php');
     808            $added_indices = true;
     809            add_clean_index( $wpdb->comments, 'comment_type'  );
     810            add_clean_index( $wpdb->comments, 'comment_karma' );
     811            add_clean_index( $wpdb->comments, 'comment_agent' );
     812        }
     813       
     814        // Get LJ comments, which haven't been threaded yet, 5000 at a time and thread them
     815        while ( $comments = $wpdb->get_results( "SELECT comment_ID, comment_agent FROM {$wpdb->comments} WHERE comment_type = 'livejournal' AND comment_agent != '0' LIMIT 5000", OBJECT ) ) {
     816            foreach ( $comments as $comment ) {
     817                $wpdb->update( $wpdb->comments,
     818                                array( 'comment_parent' => $this->get_wp_comment_ID( $comment->comment_agent ), 'comment_type' => 'livejournal-done' ),
     819                                array( 'comment_ID' => $comment->comment_ID ) );
     820            }
     821            wp_cache_flush();
     822            $wpdb->flush();
     823        }
     824       
     825        // Revert the comments table back to normal and optimize it to reclaim space
     826        if ( $added_indices ) {
     827            drop_index( $wpdb->comments, 'comment_type'  );
     828            drop_index( $wpdb->comments, 'comment_karma' );
     829            drop_index( $wpdb->comments, 'comment_agent' );
     830            $wpdb->query( "OPTIMIZE TABLE {$wpdb->comments}" );
     831        }
     832       
     833        // Clean up database and we're out
     834        $this->cleanup();
     835        do_action( 'import_done', 'livejournal' );
     836        if ( $imported_comments > 1 )
     837            echo '<p>' . sprintf( __( "Successfully re-threaded %d comments." ), number_format( $imported_comments ) ) . '</p>';
     838        echo '<h3>';
     839        printf( __( 'All done. <a href="%s">Have fun!</a>' ), get_option( 'home' ) );
     840        echo '</h3>';
    1024841        echo '</div>';
    1025842    }
    … …  
    1035852       
    1036853        return $str;
    1037     }
    1038 
    1039     // Automatically submit the form with #id to continue the process
    1040     // Hide any submit buttons to avoid people clicking them
    1041     // Display a countdown in the element indicated by $msg for "Continuing in x"
    1042     function auto_ajax( $id = 'ljapi-next-form', $msg = 'auto-message', $seconds = 5 ) {
    1043         ?><script type="text/javascript">
    1044             next_counter = <?php echo $seconds ?>;
    1045             jQuery(document).ready(function(){
    1046                 ljapi_msg();
    1047             });
    1048            
    1049             function ljapi_msg() {
    1050                 str = '<?php _e( "Continuing in %d" ) ?>';
    1051                 jQuery( '#<?php echo $msg ?>' ).text( str.replace( /%d/, next_counter ) );
    1052                 if ( next_counter <= 0 ) {
    1053                     if ( jQuery( '#<?php echo $id ?>' ).length ) {
    1054                         jQuery( "#<?php echo $id ?> input[type='submit']" ).hide();
    1055                         jQuery.ajaxSetup({'timeout':3600000});
    1056                         str = '<?php _e( "Processing next file." ) ?> <img src="images/loading-publish.gif" alt="" id="processing" align="top" />';
    1057                         jQuery( '#<?php echo $msg ?>' ).html( str );
    1058                         jQuery('#ljapi-status').load(ajaxurl, {'action':'lj-importer',
    1059                                                                 'step':jQuery('#step').val(),
    1060                                                                 '_wpnonce':'<?php echo wp_create_nonce( 'lj-api-import' ) ?>',
    1061                                                                 '_wp_http_referer':'<?php echo $_SERVER['REQUEST_URI'] ?>'});
    1062                         return;
    1063                     }
    1064                 }
    1065                 next_counter = next_counter - 1;
    1066                 setTimeout('ljapi_msg()', 1000);
    1067             }
    1068         </script><?php
    1069854    }
    1070855   
    … …  
    1095880        </script><?php
    1096881    }
    1097 
    1098     // Remove all options used during import process
     882   
     883    // Automatically submit the form with #id to continue the process
     884    // Hide any submit buttons to avoid people clicking them
     885    // Display a countdown in the element indicated by $msg for "Continuing in x"
     886    function auto_ajax( $id = 'ljapi-next-form', $msg = 'auto-message', $seconds = 5 ) {
     887        ?><script type="text/javascript">
     888            next_counter = <?php echo $seconds ?>;
     889            jQuery(document).ready(function(){
     890                ljapi_msg();
     891            });
     892           
     893            function ljapi_msg() {
     894                str = '<?php _e( "Continuing in %d" ) ?>';
     895                jQuery( '#<?php echo $msg ?>' ).text( str.replace( /%d/, next_counter ) );
     896                if ( next_counter <= 0 ) {
     897                    if ( jQuery( '#<?php echo $id ?>' ).length ) {
     898                        jQuery( "#<?php echo $id ?> input[type='submit']" ).hide();
     899                        jQuery.ajaxSetup({'timeout':3600000});
     900                        str = '<?php _e( "Processing next batch." ) ?> <img src="images/loading-publish.gif" alt="" id="processing" align="top" />';
     901                        jQuery( '#<?php echo $msg ?>' ).html( str );
     902                        jQuery('#ljapi-status').load(ajaxurl, {'action':'lj-importer',
     903                                                                'step':jQuery('#step').val(),
     904                                                                '_wpnonce':'<?php echo wp_create_nonce( 'lj-api-import' ) ?>',
     905                                                                '_wp_http_referer':'<?php echo $_SERVER['REQUEST_URI'] ?>'});
     906                        return;
     907                    }
     908                }
     909                next_counter = next_counter - 1;
     910                setTimeout('ljapi_msg()', 1000);
     911            }
     912        </script><?php
     913    }
     914
     915    // Remove all options used during import process and
     916    // set wp_comments entries back to "normal" values
    1099917    function cleanup() {
     918        global $wpdb;
     919       
    1100920        delete_option( 'ljapi_username' );
    1101921        delete_option( 'ljapi_password' );
    … …  
    1112932        delete_option( 'ljapi_highest_id' );
    1113933        delete_option( 'ljapi_highest_comment_id' );
    1114         delete_option( 'ljapi_comment_xml_files' );
    1115         delete_option( 'ljapi_comment_xml_files_count' );
    1116         delete_option( 'ljapi_comment_cache_files' );
    1117         delete_option( 'ljapi_comment_cache_files_count' );
    1118         delete_option( 'ljapi_comment_threaded_files' );
    1119         delete_option( 'ljapi_comment_threaded_files_count' );
     934        delete_option( 'ljapi_comment_batch' );
    1120935        delete_option( 'ljapi_step' );
    1121     }
    1122    
    1123     // Dump a string to a log file (appends to existing file)
    1124     function log( $string, $name ) {
    1125         return; // remove this to enable "debugging" output to files in /wp-content/ljimport
    1126         $path = wp_upload_dir();
    1127         $path = $path['path'];
    1128         if ( get_option( 'uploads_use_yearmonth_folders' ) )
    1129             $path = substr( $path, 0, -8 );
    1130 
    1131         if ( !is_dir( $path . '/ljimport' ) )
    1132             mkdir( $path . '/ljimport' );
    1133            
    1134         $fh = @fopen( $path . '/ljimport/' . $name, 'a' );
    1135         if ( $fh ) {
    1136             if ( is_array( $string ) || is_object( $string ) )
    1137                 fwrite( $fh, var_export( $string, true ) . "\n\n" );
    1138             else
    1139                 fwrite( $fh, $string . "\n\n" );
    1140             fclose( $fh );
    1141         }
    1142     }
    1143    
    1144     function write_file( $data, $name, $id, $mode = 'a' ) {
    1145         if ( empty( $this->pointers[$id] ) )
    1146             $this->pointers[$id] = @fopen( $name, $mode );
    1147         if ( $this->pointers[$id] )
    1148             return fwrite( $this->pointers[$id], $data );
    1149         return false;
    1150     }
    1151    
    1152     function full_path( $basename ) {
    1153         $uploads = wp_upload_dir();
    1154         return $uploads['path'] . '/' . $basename;
    1155     }
    1156    
    1157     function close_file_pointers() {
    1158         foreach ( $this->pointers as $p )
    1159             @fclose( $p );
    1160     }
    1161 
     936       
     937        $wpdb->update( $wpdb->comments,
     938                        array( 'comment_karma' => 0, 'comment_agent' => 'WP LJ Importer', 'comment_type' => '' ),
     939                        array( 'comment_type' => 'livejournal-done' ) );
     940        $wpdb->update( $wpdb->comments,
     941                        array( 'comment_karma' => 0, 'comment_agent' => 'WP LJ Importer', 'comment_type' => '' ),
     942                        array( 'comment_type' => 'livejournal' ) );
     943    }
     944   
    1162945    function LJ_API_Import() {
    1163946        $this->__construct();
Note: See TracChangeset for help on using the changeset viewer.