Index: livejournal.php =================================================================== --- livejournal.php (revision 10528) +++ livejournal.php (working copy) @@ -1,4 +1,5 @@

-

+

@@ -228,6 +226,10 @@

NOTE: If the import process is interrupted for any reason, come back to this page and it will continue from where it stopped automatically.' ) ?>

+ + @@ -248,7 +250,6 @@ do { $lastsync = date( 'Y-m-d H:i:s', strtotime( get_option( 'ljapi_lastsync' ) ) ); $synclist = $this->lj_ixr( 'syncitems', array( 'ver' => 1, 'lastsync' => $lastsync ) ); - $this->log( $synclist, 'ljimport-items-' . $total . '.txt' ); // Keep track of if we've downloaded everything $total = $synclist['total']; @@ -261,15 +262,16 @@ $lastsync = $event['time']; } } - - update_option( 'ljapi_sync_item_times', $sync_item_times ); - update_option( 'ljapi_total', $total ); - update_option( 'ljapi_count', $count ); - update_option( 'ljapi_lastsync', $lastsync ); } while ( $total > $count ); // endwhile - all post meta is cached locally - $this->log( $sync_item_times, 'ljimport-post-mod-times.txt' ); + update_option( 'ljapi_sync_item_times', $sync_item_times ); + update_option( 'ljapi_total', $total ); + update_option( 'ljapi_count', $count ); + update_option( 'ljapi_lastsync', $lastsync ); + + echo '

' . __( 'Post metadata has been downloaded, proceeding with posts...' ) . '

'; + echo '
    '; $imported_count = (int) get_option( 'ljapi_imported_count' ); @@ -285,9 +287,9 @@ 'selecttype' => 'syncitems', 'lineendings' => 'pc', 'lastsync' => $lastsync ) ); - $this->log( $itemlist, 'ljimport-posts-' . $imported_count . '.txt' ); if ( is_wp_error( $itemlist ) ) return $itemlist; + if ( $num = count( $itemlist['events'] ) ) { foreach ( $itemlist['events'] as $event ) { $imported_count++; @@ -312,9 +314,10 @@ return; $user = wp_get_current_user(); - $post_author = $user->ID; - $post_status = ( 'private' == trim( $post['security'] ) ) ? 'private' : 'publish'; // Only me - $post_password = ( 'usemask' == trim( $post['security'] ) ) ? $this->protected_password : ''; // "Friends" via password + $post_author = $user->ID; + $post['security'] = !empty( $post['security'] ) ? $post['security'] : ''; + $post_status = ( 'private' == trim( $post['security'] ) ) ? 'private' : 'publish'; // Only me + $post_password = ( 'usemask' == trim( $post['security'] ) ) ? $this->protected_password : ''; // "Friends" via password // For some reason, LJ sometimes sends a date as "2004-04-1408:38:00" (no space btwn date/time) $post_date = $post['eventtime']; @@ -322,7 +325,7 @@ $post_date = substr( $post_date, 0, 10 ) . ' ' . substr( $post_date, 10 ); // Cleaning up and linking the title - $post_title = trim( $post['subject'] ); + $post_title = isset( $post['subject'] ) ? trim( $post['subject'] ) : ''; $post_title = $this->translate_lj_user( $post_title ); // Translate it, but then we'll strip the link $post_title = strip_tags( $post_title ); // Can't have tags in the title in WP $post_title = $wpdb->escape( $post_title ); @@ -365,7 +368,6 @@ } $postdata['post_ID'] = $post_id; $postdata['lj_itemid'] = $post['itemid']; - $this->log( $postdata, 'ljimport-post-' . $post_id . '.txt' ); // Handle all the metadata for this post $this->insert_postmeta( $post_id, $post ); @@ -404,34 +406,41 @@ } } - // Loops through and gets comment meta and content from LJ in batches - // Writes raw XML files to disk for later processing - function download_comments() { + // Set up a session (authenticate) with LJ + function getsession() { // Get a session via XMLRPC $cookie = $this->lj_ixr( 'sessiongenerate', array( 'ver' => 1, 'expiration' => 'short' ) ); + return new WP_Http_Cookie( array( 'name' => 'ljsession', 'value' => $cookie['ljsession'] ) ); + } + + // Loops through and gets comment meta from LJ in batches + function download_comment_meta() { + $cookie = $this->getsession(); - // Comment Meta - // Load previous state (if any) $this->usermap = (array) get_option( 'ljapi_usermap' ); - $maxid = (int) get_option( 'ljapi_maxid' ) || 1; - $highest_id = (int) get_option( 'ljapi_highest_id' ); - - // Snoopy is required to handle the cookie - $this->snoop = new Snoopy(); - $this->snoop->cookies = $cookie; + $maxid = get_option( 'ljapi_maxid' ) ? get_option( 'ljapi_maxid' ) : 1; + $highest_id = get_option( 'ljapi_highest_id' ) ? get_option( 'ljapi_highest_id' ) : 0; // We need to loop over the metadata request until we have it all while ( $maxid > $highest_id ) { // Now get the meta listing - if ( !$this->snoop->fetch( $this->comments_url . '?get=comment_meta&startid=' . ( $highest_id + 1 ) ) ) - return new WP_Error( 'Snoopy', __( 'Failed to retrieve comment meta information from LiveJournal. Please try again soon.' ) ); - - // Snoopy doesn't provide an accessor for results... - $results = $this->snoop->results; + $results = wp_remote_get( $this->comments_url . '?get=comment_meta&startid=' . ( $highest_id + 1 ), + array( 'cookies' => array( $cookie ), 'timeout' => 20 ) ); + if ( is_wp_error( $results ) ) + return new WP_Error( 'comment_meta', __( 'Failed to retrieve comment meta information from LiveJournal. Please try again soon.' ) ); + $results = wp_remote_retrieve_body( $results ); + // Get the maxid so we know if we have them all yet preg_match( '|(\d+)|', $results, $matches ); + if ( 0 == $matches[1] ) { + // No comment meta = no comments for this journal + echo '

    ' . __( 'You have no comments to import!' ) . '

    '; + update_option( 'ljapi_highest_id', 1 ); + update_option( 'ljapi_highest_comment_id', 1 ); + return false; // Bail out of comment importing entirely + } $maxid = !empty( $matches[1] ) ? $matches[1] : $maxid; // Parse comments and get highest id available @@ -446,112 +455,75 @@ foreach ( $matches[1] as $count => $userid ) $this->usermap[$userid] = $matches[2][$count]; // need this in memory for translating ids => names - update_option( 'ljapi_usermap', $this->usermap ); - update_option( 'ljapi_maxid', $maxid ); - update_option( 'ljapi_highest_id', $highest_id ); + wp_cache_flush(); } // endwhile - should have seen all comment meta at this point + + update_option( 'ljapi_usermap', $this->usermap ); + update_option( 'ljapi_maxid', $maxid ); + update_option( 'ljapi_highest_id', $highest_id ); + echo '

    ' . __( ' Comment metadata downloaded successfully, proceeding with posts...' ) . '

    '; - // Download Comment XML - - // Load previous state (if any) - $highest_id = (int) get_option( 'ljapi_highest_comment_id' ); - $comment_xml_files = get_option( 'ljapi_comment_xml_files' ); - if ( !is_array( $comment_xml_files ) ) { - update_option( 'ljapi_comment_xml_files', array() ); - $comment_xml_files = array(); - } - - echo '
      '; - - // And now request the actual comments, and keep going until we have them all - while ( $maxid > $highest_id ) { - // Get a batch of comments, using the highest_id we've already got as a starting point - if ( !$this->snoop->fetch( $this->comments_url . '?get=comment_body&startid=' . ( $highest_id + 1 ) ) ) - return new WP_Error( 'Snoopy', __( 'Failed to retrieve comment bodies from LiveJournal. Please try again soon.' ) ); - - // Get the highest post ID in this batch (required for loop control) - $results = $this->snoop->results; - preg_match_all( '| $highest_id ) - $highest_id = $comments[1][$r]; - } - - // $this->snoop-results is where the actual response is stored - $this->log( $this->snoop->results, 'ljimport-comment-bodies-' . $highest_id . '.txt' ); - - // Store in uploads dir. Can't use *.xml because it's not allowed - $results = wp_upload_bits( 'raw-comments-' . $highest_id . '.txt', null, $results ); - if ( !empty( $results['error'] ) ) - return new WP_Error( 'xml', $results['error'] ); - $comment_xml_files[] = $results['file']; - - echo '
    1. ' . sprintf( __( 'Downloaded %s' ), basename( $results['file'] ) ) . '
      2. '; - ob_flush(); flush(); - - $comment_xml_files = array_unique( $comment_xml_files ); - update_option( 'ljapi_comment_xml_files', $comment_xml_files ); - update_option( 'ljapi_comment_xml_files_count', count( $comment_xml_files ) ); - } - // endwhile - all comments downloaded and ready for bulk processing - - echo '
    '; - return true; } - function parse_comment_xml( $xml_file ) { - if ( !is_file( $xml_file ) || !is_readable( $xml_file ) ) - return new WP_Error( 'file', sprintf( __( 'Could not access comment XML file: %s'), $filename ) ); - - // Get content from file - $xml = @file_get_contents( $xml_file ); - - $cache_files = get_option( 'ljapi_comment_cache_files' ); - if ( !is_array( $cache_files ) ) - $cache_files = array(); + // Downloads actual comment bodies from LJ + // Inserts them all directly to the DB, with additional info stored in "spare" fields + function download_comment_bodies() { + global $wpdb; + $cookie = $this->getsession(); - // Parse XML into comments - preg_match_all( '||iUs', $xml, $matches ); - unset( $xml ); - for ( $c = 0; $c < count( $matches[0] ); $c++ ) { - $comment = $matches[0][$c]; + // Load previous state (if any) + $this->usermap = (array) get_option( 'ljapi_usermap' ); + $maxid = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1; + $highest_id = (int) get_option( 'ljapi_highest_comment_id' ); + $loop = 0; + while ( $maxid > $highest_id && $loop < 5 ) { // We do 5 loops per call to avoid memory limits + $loop++; - // Filter out any captured, deleted comments (nothing useful to import) - $comment = preg_replace( '||is', '', $comment ); + // Get a batch of comments, using the highest_id we've already got as a starting point + $results = wp_remote_get( $this->comments_url . '?get=comment_body&startid=' . ( $highest_id + 1 ), + array( 'cookies' => array( $cookie ), 'timeout' => 20 ) ); + if ( is_wp_error( $results ) ) + return new WP_Error( 'comment_bodies', __( 'Failed to retrieve comment bodies from LiveJournal. Please try again soon.' ) ); - // Parse this comment into an array - $comment = $this->parse_comment( $comment ); - if ( empty( $comment['comment_post_ID'] ) ) - continue; + $results = wp_remote_retrieve_body( $results ); - // Add this comment to the appropriate cache file - $filename = $this->full_path( 'ljimport-comments-' . $comment['comment_post_ID'] . '.php' ); - if ( $this->write_file( '' . "\n", - $filename, - $comment['comment_post_ID'], - 'a' ) ) - { - // Keep track of files used - $cache_files[] = $filename; + // Parse out each comment and insert directly + preg_match_all( '||iUs', $results, $matches ); + for ( $c = 0; $c < count( $matches[0] ); $c++ ) { + // Keep track of highest id seen + if ( $matches[1][$c] > $highest_id ) { + $highest_id = $matches[1][$c]; + update_option( 'ljapi_highest_comment_id', $highest_id ); + } + + $comment = $matches[0][$c]; + + // Filter out any captured, deleted comments (nothing useful to import) + $comment = preg_replace( '||is', '', $comment ); + + // Parse this comment into an array and insert + $comment = $this->parse_comment( $comment ); + $id = wp_insert_comment( $comment ); + + // Clear cache + clean_comment_cache( $id ); } + + // Clear cache to preseve memory + wp_cache_flush(); } + // endwhile - all comments downloaded and ready for bulk processing - // Update list of files in the DB - sort( $cache_files ); - $cache_files = array_unique( $cache_files ); - update_option( 'ljapi_comment_cache_files', $cache_files ); - update_option( 'ljapi_comment_cache_files_count', count( $cache_files ) ); - $this->close_file_pointers(); + // Counter just used to show progress to user + update_option( 'ljapi_comment_batch', ( (int) get_option( 'ljapi_comment_batch' ) + 1 ) ); - // Don't need this XML file anymore - unlink( $xml_file ); - return true; } + // Takes a block of XML and parses out all the elements of the comment function parse_comment( $comment ) { global $wpdb; @@ -562,17 +534,19 @@ preg_match( '| jitemid=\'(\d+)\'|i', $attribs[1], $matches ); $lj_comment_post_ID = $matches[1]; preg_match( '| posterid=\'(\d+)\'|i', $attribs[1], $matches ); - $comment_author_ID = $matches[1]; - preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches ); - $lj_comment_parent = $matches[1]; - preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches ); - $lj_comment_state = !empty( $matches[1] ) ? $matches[1] : 'A'; + $comment_author_ID = isset( $matches[1] ) ? $matches[1] : 0; + preg_match( '| parentid=\'(\d+)\'|i', $attribs[1], $matches ); // optional + $lj_comment_parent = isset( $matches[1] ) ? $matches[1] : 0; + preg_match( '| state=\'([SDFA])\'|i', $attribs[1], $matches ); // optional + $lj_comment_state = isset( $matches[1] ) ? $matches[1] : 'A'; // Clean up "subject" - this will become the first line of the comment in WP preg_match( '|(.*)|is', $comment, $matches ); - $comment_subject = $wpdb->escape( trim( $matches[1] ) ); - if ( 'Re:' == $comment_subject ) - $comment_subject = ''; + if ( isset( $matches[1] ) ) { + $comment_subject = $wpdb->escape( trim( $matches[1] ) ); + if ( 'Re:' == $comment_subject ) + $comment_subject = ''; + } // Get the body and HTMLize it preg_match( '|(.*)|is', $comment, $matches ); @@ -590,11 +564,11 @@ $comment_date = trim( str_replace( array( 'T', 'Z' ), ' ', $matches[1] ) ); // Grab IP if available - preg_match( '|(.*)|i', $comment, $matches ); - $comment_author_IP = $matches[1]; + preg_match( '|(.*)|i', $comment, $matches ); // optional + $comment_author_IP = isset( $matches[1] ) ? $matches[1] : ''; // Try to get something useful for the comment author, especially if it was "my" comment - $author = ( substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' || empty( $comment_author_ID ) ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID]; + $author = ( empty( $comment_author_ID ) || empty( $this->usermap[$comment_author_ID] ) || substr( $this->usermap[$comment_author_ID], 0, 4 ) == 'ext_' ) ? __( 'Anonymous' ) : $this->usermap[$comment_author_ID]; if ( get_option( 'ljapi_username' ) == $author ) { $user = wp_get_current_user(); $user_id = $user->ID; @@ -613,13 +587,16 @@ 'comment_post_ID' => $this->get_wp_post_ID( $lj_comment_post_ID ), 'comment_author' => $author, 'comment_author_url' => $url, + 'comment_author_email' => '', 'comment_content' => $comment_content, 'comment_date' => $comment_date, 'comment_author_IP' => ( !empty( $comment_author_IP ) ? $comment_author_IP : '' ), 'comment_approved' => ( in_array( $lj_comment_state, array( 'A', 'F' ) ) ? 1 : 0 ), - 'comment_agent' => 'WP LJ Importer', - 'user_id' => $user_id - ); + 'comment_karma' => $lj_comment_ID, // Need this and next value until rethreading is done + 'comment_agent' => $lj_comment_parent, + 'comment_type' => 'livejournal', // Custom type, so we can find it later for processing + 'user_ID' => $user_id + ); } @@ -627,126 +604,18 @@ function get_wp_post_ID( $post ) { global $wpdb; if ( empty( $this->postmap[$post] ) ) - $this->postmap[$post] = $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) ); + $this->postmap[$post] = (int) $wpdb->get_var( $wpdb->prepare( "SELECT post_id FROM $wpdb->postmeta WHERE meta_key = 'lj_itemid' AND meta_value = %d", $post ) ); return $this->postmap[$post]; } - // Re-build the threading within a single cache file - function thread_comments( $filename ) { - if ( !is_file( $filename ) || !is_readable( $filename ) ) - return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) ); - - $comments = array(); - @include( $filename ); - $this->comments = $comments; - unset( $comments ); - if ( !is_array( $this->comments ) ) - $this->comments = array(); - - $count = count( $this->comments ); - for ( $c = 0; $c < $count; $c++ ) { - // Skip anything that's not "top-level" for now - if ( 0 != $this->comments[$c]['lj_comment_parent'] ) - continue; - $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] ); - } - - // Remove anything that's not supposed to be at top level - $top_comments = array(); - for ( $c = 0; $c < $count; $c++ ) { - if ( 0 == $this->comments[$c]['lj_comment_parent'] ) { - $top_comments[] = $this->comments[$c]; - } - } - - // Write back to file - @unlink( $filename ); - $this->write_file( '', $filename, $count, 'w' ); - unset( $top_comments ); - $this->close_file_pointers(); - - // Reference this file as being threaded - $files = get_option( 'ljapi_comment_threaded_files' ); - $files[] = $filename; - array_unique( $files ); - update_option( 'ljapi_comment_threaded_files', $files ); - update_option( 'ljapi_comment_threaded_files_count', count( $files ) ); - - return true; + // Gets the comment_ID that a LJ comment has been saved as within WP + function get_wp_comment_ID( $comment ) { + global $wpdb; + if ( empty( $this->commentmap[$comment] ) ) + $this->commentmap[$comment] = $wpdb->get_var( $wpdb->prepare( "SELECT comment_ID FROM $wpdb->comments WHERE comment_karma = %d", $comment ) ); + return $this->commentmap[$comment]; } - - function get_child_comments( $id ) { - $children = array(); - $count = count( $this->comments ); - for ( $c = 0; $c < $count; $c++ ) { - // This comment is a child of the $id - if ( $id == $this->comments[$c]['lj_comment_parent'] ) { - $this->comments[$c]['children'] = $this->get_child_comments( $this->comments[$c]['lj_comment_ID'] ); - $children[] = $this->comments[$c]; - } - } - return $children; - } - - // Inserts the contents of each cache file (should be threaded already) - function insert_comments( $filename ) { - echo '
      '; - - if ( !is_file( $filename ) || !is_readable( $filename ) ) - return new WP_Error( 'File', __( sprintf( 'Cannot access file %s', $filename ) ) ); - - $comments = array(); - @include( $filename ); - $this->comments = $comments; - unset( $comments ); - if ( !is_array( $this->comments ) ) - $this->comments = array(); - $count = count( $this->comments ); - for ( $c = 0; $c < $count; $c++ ) { - $comment =& $this->comments[$c]; - echo '
    1. '; - printf( __( 'Imported comment from %s on %s' ), $comment['comment_author'], $comment['comment_date'] ); - - $id = wp_insert_comment( $comment ); - $comment['comment_ID'] = $id; - if ( count( $comment['children'] ) ) { - _e( ' and replies:' ); - $this->insert_child_comments( $comment['children'], $id ); - } - - echo '
      2. '; - } - - // Remove the file now that we're done with it - @unlink( $filename ); - - echo '
    '; - - return true; - } - - function insert_child_comments( &$comments, $parent ) { - echo '
      '; - $count = count( $comments ); - for ( $c = 0; $c < $count; $c++ ) { - $comment =& $comments[$c]; - $comment['comment_parent'] = $parent; - echo '
    1. '; - printf( __( 'Imported reply from %s on %s' ), $comment['comment_author'], $comment['comment_date'] ); - - $id = wp_insert_comment( $comment ); - $comment['comment_ID'] = $id; - if ( count( $comment['children'] ) ) { - _e( ' and replies:' ); - $this->insert_child_comments( $comment['children'], $id ); - } - - echo '
      2. '; - } - echo '
    '; - } - function lj_ixr() { if ( $challenge = $this->ixr->query( 'LJ.XMLRPC.getchallenge' ) ) { $challenge = $this->ixr->getResponse(); @@ -767,7 +636,6 @@ if ( $this->ixr->query( 'LJ.XMLRPC.' . $method, $params ) ) { return $this->ixr->getResponse(); } else { - $this->log( $this->ixr->message, 'ljimport-error-' . $method . '.txt' ); return new WP_Error( 'IXR', __( 'XML-RPC Request Failed - ' ) . $this->ixr->getErrorCode() . ': ' . $this->ixr->getErrorMessage() ); } } @@ -788,12 +656,10 @@ $this->greet(); break; case 1 : - case 2 : $this->ixr = new IXR_Client( $this->ixr_url ); // Intentional no break + case 2 : case 3 : - case 4 : - case 5 : check_admin_referer( 'lj-api-import' ); $result = $this->{ 'step' . $step }(); if ( is_wp_error( $result ) ) @@ -837,6 +703,8 @@ $login = $this->lj_ixr( 'login' ); if ( is_wp_error( $login ) ) { if ( 100 == $this->ixr->getErrorCode() || 101 == $this->ixr->getErrorCode() ) { + delete_option( 'ljapi_username' ); + delete_option( 'ljapi_password' ); ?>

    @@ -848,11 +716,9 @@ } // Set up some options to avoid them autoloading (these ones get big) - add_option( 'ljapi_sync_item_times', '', '', 'no' ); - add_option( 'ljapi_usermap', '', '', 'no' ); - add_option( 'ljapi_comment_xml_files', '', '', 'no' ); - add_option( 'ljapi_comment_cache_files', '', '', 'no' ); - add_option( 'ljapi_comment_threaded_files', '', '', 'no' ); + add_option( 'ljapi_sync_item_times', '', '', 'no' ); + add_option( 'ljapi_usermap', '', '', 'no' ); + update_option( 'ljapi_comment_batch', 0 ); echo '

    ' . __( 'Importing Posts' ) . '

    '; echo '

    ' . __( "We're downloading and importing all your LiveJournal posts..." ) . '

    '; @@ -873,7 +739,7 @@ } } - echo '

    ' . __( "Your posts have all been imported, but wait - there's more! Now we need to process & import your comments." ) . '

    '; + echo '

    ' . __( "Your posts have all been imported, but wait - there's more! Now we need to download & import your comments." ) . '

    '; echo $this->next_step( 2, __( 'Download my comments »' ) ); $this->auto_submit(); } @@ -884,143 +750,94 @@ update_option( 'ljapi_step', 2 ); $this->username = get_option( 'ljapi_username' ); $this->password = get_option( 'ljapi_password' ); + $this->ixr = new IXR_Client( $this->ixr_url ); + echo '
    '; echo '

    ' . __( 'Downloading Comments' ) . '

    '; - echo '

    ' . __( 'Now we will download your comments so we can process and import them...' ) . '

    '; + echo '

    ' . __( 'Now we will download your comments so we can import them (this could take a long time if you have lots of comments)...' ) . '

    '; ob_flush(); flush(); - $result = $this->download_comments(); - if ( is_wp_error( $result ) ) - return $result; - - echo '

    ' . __( 'Your comments have all been downloaded to this server now, so we can process them and get them ready for importing.' ) . '

    '; - echo $this->next_step( 3, __( 'Process my comment files »' ) ); - $this->auto_submit(); - } - - // Parse XML into comment cache files - function step3() { - - set_time_limit( 0 ); - update_option( 'ljapi_step', 3 ); - - $this->usermap = get_option( 'ljapi_usermap' ); - - echo '
    '; - echo '

    ' . __( 'Parsing Comments' ) . '

    '; - echo '

    ' . __( 'Time to clean up your comments and get them into a format WordPress understands...' ) . '

    '; - ob_flush(); flush(); - - $files = get_option( 'ljapi_comment_xml_files' ); - if ( count( $files ) ) { - $file = array_pop( $files ); - - $result = $this->parse_comment_xml( $file ); + if ( !get_option( 'ljapi_usermap' ) ) { + // We haven't downloaded meta yet, so do that first + $result = $this->download_comment_meta(); if ( is_wp_error( $result ) ) return $result; - - update_option( 'ljapi_comment_xml_files', $files ); } - - if ( count( $files ) ) { - ?> -
    -

    - - -

    -
    - auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> - download_comment_bodies(); + if ( is_wp_error( $result ) ) + return $result; + + $maxid = get_option( 'ljapi_maxid' ) ? (int) get_option( 'ljapi_maxid' ) : 1; + $highest_id = (int) get_option( 'ljapi_highest_comment_id' ); + if ( $maxid > $highest_id ) { + ?> +
    +

    approximately %d' ), get_option( 'ljapi_comment_batch' ), ( $maxid / 5000 ) ) ?>

    + + +

    +
    + auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> + ' . __( 'Yay, we finished processing all of your comment files! Now we need to re-build your conversation threads.' ) . '

    '; - echo $this->next_step( 4, __( 'Thread my comments »' ) ); + echo '

    ' . __( 'Your comments have all been imported now, but we still need to rebuild your conversation threads.' ) . '

    '; + echo $this->next_step( 3, __( 'Rebuild my comment threads »' ) ); $this->auto_submit(); } echo '
    '; } - - // Thread comments within their cache files - function step4() { + + // Re-thread comments already in the DB + function step3() { + global $wpdb; set_time_limit( 0 ); - update_option( 'ljapi_step', 4 ); + update_option( 'ljapi_step', 3 ); echo '
    '; echo '

    ' . __( 'Threading Comments' ) . '

    '; - echo '

    ' . __( 'Re-building your conversation threads ready for import...' ) . '

    '; + echo '

    ' . __( 'We are now re-building the threading of your comments (this can also take a while if you have lots of comments)...' ) . '

    '; ob_flush(); flush(); - $files = get_option( 'ljapi_comment_cache_files' ); - if ( count( $files ) ) { - $file = array_pop( $files ); - - $result = $this->thread_comments( $file ); - if ( is_wp_error( $result ) ) - return $result; - - update_option( 'ljapi_comment_cache_files', $files ); + // Only bother adding indexes if they have over 5000 comments (arbitrary number) + $imported_comments = $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->comments} WHERE comment_type = 'livejournal'" ); + $added_indices = false; + if ( 5000 < $imported_comments ) { + include_once(ABSPATH . 'wp-admin/includes/upgrade.php'); + $added_indices = true; + add_clean_index( $wpdb->comments, 'comment_type' ); + add_clean_index( $wpdb->comments, 'comment_karma' ); + add_clean_index( $wpdb->comments, 'comment_agent' ); } - if ( count( $files ) ) { - ?> -
    -

    - - -

    -
    - auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> - ' . __( "Alrighty, your comments are all threaded. There's just one last step -- time to actually import them all now!" ) . '

    '; - echo '

    ' . __( 'This last part in particular can take a really long time if you have a lot of comments. You might want to go and do something else while you wait.' ) . '

    '; - echo $this->next_step( 5, __( 'Import my threaded comments into WordPress »' ) ); - $this->auto_submit(); + // Get LJ comments, which haven't been threaded yet, 5000 at a time and thread them + while ( $comments = $wpdb->get_results( "SELECT comment_ID, comment_agent FROM {$wpdb->comments} WHERE comment_type = 'livejournal' AND comment_agent != '0' LIMIT 5000", OBJECT ) ) { + foreach ( $comments as $comment ) { + $wpdb->update( $wpdb->comments, + array( 'comment_parent' => $this->get_wp_comment_ID( $comment->comment_agent ), 'comment_type' => 'livejournal-done' ), + array( 'comment_ID' => $comment->comment_ID ) ); + } + wp_cache_flush(); + $wpdb->flush(); } - echo '
    '; - } - - // Import comments from cache files into WP - function step5() { - set_time_limit( 0 ); - update_option( 'ljapi_step', 5 ); - - echo '
    '; - echo '

    ' . __( 'Importing Comments' ) . '

    '; - echo '

    ' . __( 'This is the big one -- we are now inserting your comment threads into WordPress...' ) . '

    '; - - $files = get_option( 'ljapi_comment_threaded_files' ); - echo '

    ' . sprintf( __( 'Importing cache file %d of %d' ), ( get_option( 'ljapi_comment_threaded_files_count' ) - count( $files ) + 1 ), get_option( 'ljapi_comment_threaded_files_count' ) ) . '

    '; - ob_flush(); flush(); - - if ( count( $files ) ) { - $file = array_pop( $files ); - - $result = $this->insert_comments( $file ); - if ( is_wp_error( $result ) ) - return $result; - - update_option( 'ljapi_comment_threaded_files', $files ); + // Revert the comments table back to normal and optimize it to reclaim space + if ( $added_indices ) { + drop_index( $wpdb->comments, 'comment_type' ); + drop_index( $wpdb->comments, 'comment_karma' ); + drop_index( $wpdb->comments, 'comment_agent' ); + $wpdb->query( "OPTIMIZE TABLE {$wpdb->comments}" ); } - if ( count( $files ) ) { - ?> -
    - - -

    -
    - auto_ajax( 'ljapi-auto-repost', 'auto-message', 0 ); ?> - cleanup(); - do_action( 'import_done', 'livejournal' ); - echo '

    '; - printf( __( 'All done. Have fun!' ), get_option( 'home' ) ); - echo '

    '; - } + // Clean up database and we're out + $this->cleanup(); + do_action( 'import_done', 'livejournal' ); + if ( $imported_comments > 1 ) + echo '

    ' . sprintf( __( "Successfully re-threaded %d comments." ), number_format( $imported_comments ) ) . '

    '; + echo '

    '; + printf( __( 'All done. Have fun!' ), get_option( 'home' ) ); + echo '

    '; echo '
    '; } @@ -1035,11 +852,10 @@ return $str; } - - // Automatically submit the form with #id to continue the process - // Hide any submit buttons to avoid people clicking them - // Display a countdown in the element indicated by $msg for "Continuing in x" - function auto_ajax( $id = 'ljapi-next-form', $msg = 'auto-message', $seconds = 5 ) { + + // Automatically submit the specified form after $seconds + // Include a friendly countdown in the element with id=$msg + function auto_submit( $id = 'ljapi-next-form', $msg = 'auto-message', $seconds = 10 ) { ?>update( $wpdb->comments, + array( 'comment_karma' => 0, 'comment_agent' => 'WP LJ Importer', 'comment_type' => '' ), + array( 'comment_type' => 'livejournal-done' ) ); + $wpdb->update( $wpdb->comments, + array( 'comment_karma' => 0, 'comment_agent' => 'WP LJ Importer', 'comment_type' => '' ), + array( 'comment_type' => 'livejournal' ) ); } - // Dump a string to a log file (appends to existing file) - function log( $string, $name ) { - return; // remove this to enable "debugging" output to files in /wp-content/ljimport - $path = wp_upload_dir(); - $path = $path['path']; - if ( get_option( 'uploads_use_yearmonth_folders' ) ) - $path = substr( $path, 0, -8 ); - - if ( !is_dir( $path . '/ljimport' ) ) - mkdir( $path . '/ljimport' ); - - $fh = @fopen( $path . '/ljimport/' . $name, 'a' ); - if ( $fh ) { - if ( is_array( $string ) || is_object( $string ) ) - fwrite( $fh, var_export( $string, true ) . "\n\n" ); - else - fwrite( $fh, $string . "\n\n" ); - fclose( $fh ); - } - } - - function write_file( $data, $name, $id, $mode = 'a' ) { - if ( empty( $this->pointers[$id] ) ) - $this->pointers[$id] = @fopen( $name, $mode ); - if ( $this->pointers[$id] ) - return fwrite( $this->pointers[$id], $data ); - return false; - } - - function full_path( $basename ) { - $uploads = wp_upload_dir(); - return $uploads['path'] . '/' . $basename; - } - - function close_file_pointers() { - foreach ( $this->pointers as $p ) - @fclose( $p ); - } - function LJ_API_Import() { $this->__construct(); }