#!/usr/bin/perl -w

use strict;
use Simple; # corrected LJ::Simple
use XML::Parser;
use POSIX;

require "$ENV{'LJHOME'}/cgi-bin/ljlib.pl";
require "ljr-defaults.pl";
require "ljr-links.pl";
require LJR::Distributed;
require LJR::unicode;
require "$ENV{'LJHOME'}/cgi-bin/talklib.pl";
require "ipics.pl";

my $err = sub {
  my %res = ();

  $res{"err"} = 1;
  $res{"errtext"} = join ("\n", @_);
  return \%res;
};
my $xmlerrt;
my $xmlerr = sub {
  my $expat = shift;

  my $cstack = "\ncallstack:";
  my $i = 0;
  while ( 1 ) {
    my $tfunc = (caller($i))[3];
    if ($tfunc && $tfunc ne "") {
      if ($tfunc !~ /\_\_ANON\_\_/) {
        $cstack .= " " . $tfunc;
      }
      $i = $i + 1;
    }
    else {
      last;
    }
  }

  $xmlerrt = join("\n", @_);
  $xmlerrt .= $cstack;
  $expat->finish();
};

my $dumpxml = sub {
  my ($xdata, $username) = @_;

  my $t = `date +"%T"`;
  $t = substr($t, 0, length($t) - 1);
  open(my $outfile, ">$ENV{'LJHOME'}/logs/err_" . $username . "_" . $t . ".xml");
  print $outfile "$xdata";
  close($outfile);
};

# kind of global variables
my $DEBUG=1;

my $got_max_commentid;
my $empty_num = 0;

my $ru; # current remote user (the one being cached and imported)
my $cmode;
my $xml_maxid = 0;
my $soft_cached_default_pics;
my $soft_cached_keyworded_pics;
my $posters_without_names; # lj bug workaround: http://rt.livejournal.org/Ticket/Display.html?id=762

sub cstatus_print {
  my $statustr = join("", @_);

  eval { LJR::Import::import_log($statustr); };

  if ($@) {
    print $@ . "\n";
  }

  if ($DEBUG) {
    eval { LJR::Import::log_print($statustr); };
    
    if ($@) {
      print $statustr . "\n";
    }
  }
}


# comment processing
my $skip_tags = qr/^(livejournal|comments|usermaps|nextid)$/; 
my $maxid = qr/^(maxid)$/;
my $comment_tag = qr/^(comment)$/;
my $usermap_tag = qr/^(usermap)$/;
my $data_tags = qr/^(subject|body|date)$/;
my $prop_tag = qr/^(property)$/;

LJ::load_props("talk");
return $err->("Can't load talkprops.") unless $LJ::CACHE_PROP{talk};
my @tprops;
foreach my $k (keys %{$LJ::CACHE_PROP{talk}}) {
  push(@tprops, $k);
}
my $tprops = join("|", @tprops);
my $known_props = qr/^($tprops)$/;

my %comment = (); # current comment structure
my $cid = ""; # current tag (inside comment tag)
my $ctext = ""; # current value of $cid


# export_comments.bml xml handling routines
sub xmlh_comment_start() {
  my $expat = shift;
  my @params = @_;
  
  if ($params[0] =~ /$skip_tags/) {
    # skip valid but meaningless tags
  }
  elsif ($cmode eq "comment_meta" && $params[0] =~ /$usermap_tag/) {
    shift @params; # skip "usermap"

    my %usermap = ();
    %usermap = @params;

    if ($usermap{id} && $usermap{user}) {
      my $r = {
        serverid => $ru->{serverid},
        userid => $usermap{id},
        username => $usermap{user},
        };

      $r = LJR::Distributed::get_cached_user($r);
      return $xmlerr->($expat, $r->{errtext}) if $r->{err};

      $r = LJR::Distributed::get_imported_user($r);
      return $xmlerr->($expat, $r->{errtext}) if $r->{err};

      $r = LJR::Distributed::set_cu_field($r, "local_commenterid", $r->{commenterid});
      return $xmlerr->($expat, $r->{errtext}) if $r->{err};

      if (!$soft_cached_default_pics->{$usermap{id}}) {
        my $iu = LJ::load_userid ($r->{local_commenterid});
#        cstatus_print(
#          "caching default userpic for " . $r->{username} .
#          " (" . $r->{local_commenterid} . ":" . $iu->{user} . ")"
#          );

        my $e = import_pics(
          $ru->{servername},
          $usermap{user},
          "",
          $iu->{user},
          "", 1);
        return $xmlerr->($expat, "importing default userpic for [" . $usermap{user} . "]:", $e->{errtext})
          if $e->{err};

        $soft_cached_default_pics->{$usermap{id}} = 1;
      }
    }
    elsif ($usermap{id} && !$usermap{user}) {
      $posters_without_names->{$usermap{id}} = 1;
    }
    else {
      return $xmlerr->($expat,
        "Unknown XML-structure: " . join (" ", @params),
        "at line " . $expat->current_line);
    }
  }
  elsif ($params[0] =~ /$comment_tag/) {
    # we're starting to process new comment
    shift @params;

    %comment = ();
    %comment = @params; # grab all comment attributes
  }
  elsif ($cmode eq "comment_body" && $params[0] =~ /$data_tags/) {
    $cid = $params[0];
    $ctext = "";
  }
  elsif ($cmode eq "comment_body" && $params[0] =~ /$prop_tag/) {
    shift @params; # skip "property"

    # skip "name" attribute name
    if (shift @params && $params[0] =~ /$known_props/) {
      $cid = $params[0];
      $ctext = "";
    }
  }
  elsif ($params[0] =~ /$maxid/) {
    $ctext = "";
  }
  else {
    return $xmlerr->($expat,
      "Unknown XML-structure: " . join (" ", @params),
      "at line " . $expat->current_line);
  }
}
sub xmlh_comment_end() {
  my $expat = shift;
  my @params = @_;

  if ($params[0] =~ /$skip_tags/) {
    # almost finished
  }
  elsif ($cmode eq "comment_meta" && $params[0] =~ /$usermap_tag/) {
    # nop
  }
  elsif ($params[0] =~ /$comment_tag/) {
    if ($cmode eq "comment_body") {
      
#      print $comment{"id"} . "\n";
#      print "COMMENT\n";
#      while ((my $k, my $v) = each(%comment)) {
#        print $k . ":" . $v . "\n";
#      }
#      print "/COMMENT\n";

      $comment{ru_id} = $ru->{ru_id};

      if (
        $comment{props} &&
        $comment{props}->{"picture_keyword"} &&
        $comment{posterid} &&
        !$soft_cached_keyworded_pics->{$comment{posterid}}->{$comment{props}->{"picture_keyword"}}
        ) {

        my $r = {
          serverid => $ru->{serverid},
          userid => $comment{posterid},
          };

        $r = LJR::Distributed::get_cached_user($r);
        return $xmlerr->($expat, $r->{errtext}) if $r->{err};

        $r = LJR::Distributed::get_imported_user($r);
        return $xmlerr->($expat, $r->{errtext} . "(userid: " . $comment{'posterid'} . ")") if $r->{err};

        $r = LJR::Distributed::get_cu_field($r, "local_commenterid");
        return $xmlerr->($expat, $r->{errtext}) if $r->{err};

        my $iu = LJ::load_userid ($r->{local_commenterid});
        #cstatus_print ("caching userpic " . $comment{props}->{"picture_keyword"} . " for " . $r->{username} . ":" . $iu->{user});

        my $e = import_pics (
          $ru->{servername},
          $r->{username},
          "",
          $iu->{user},
          $comment{props}->{"picture_keyword"},
          0);
        return $xmlerr->($expat, $e->{errtext}) if $e->{err};

        $soft_cached_keyworded_pics->{$comment{posterid}}->{$comment{props}->{"picture_keyword"}} = 1;
      }

      LJR::Links::make_ljr_hrefs(
        LJR::Links::get_server_url($ru->{"servername"}, "base"),
	$ru->{"servername"}, \$comment{body}
	);

      if ($comment{'posterid'} && $posters_without_names->{$comment{'posterid'}}) {
        $comment{'posterid'} = undef;
      }
      if (!$comment{'body'} && $comment{'state'} ne "D") {
        $comment{'body'} = "LJR::Import warning: no comment body during import.";
      }
      
      my $c = LJR::Distributed::cache_comment (\%comment);
      return $xmlerr->($expat, $c->{'errtext'}) if $c->{'err'};

      if (!$ru->{cached_comments_maxid} ||
        $comment{id} > $ru->{cached_comments_maxid}) {
        $ru->{cached_comments_maxid} = $comment{id};
      }
    }

    $got_max_commentid++;
    $empty_num = 0;
  }
  elsif ($params[0] =~ /$data_tags/) {
    $comment{$cid} = $ctext;
  }
  elsif ($params[0] =~ /$prop_tag/) {
    $comment{props}->{$cid} = $ctext;
  }
  elsif ($params[0] =~ /$maxid/) {
    $xml_maxid = $ctext;

    if ($cmode eq "comment_body" && $xml_maxid > $ru->{remote_meta_maxid}) {
      my $tmid = $got_max_commentid;
      my $txid = $xml_maxid;
      my $tempty = $empty_num;

      $got_max_commentid = $ru->{remote_meta_maxid};
      my $e = get_usermaps_cycled(
        $ru->{servername},
        $ru->{username},
        $ru->{pass},
        $got_max_commentid + 1);
      return $xmlerr->($expat, $e->{errtext}) if $e->{err};

      # restore comment_body xml-parsing mode
      $xml_maxid = $txid;
      $got_max_commentid = $tmid;
      $empty_num = $tempty;
      $cmode = "comment_body";
    }
  }
  else {
    return $xmlerr->($expat,
      "Unknown tag: " . join (" ", @params),
      "at line " . $expat->current_line
      );
  }
}
sub xmlh_comment_char() {
  my $expat = shift;
  my $tt = join("", @_);
  $ctext = $ctext . $tt;
}

sub get_usermaps_cycled {
  my ($server, $username, $pass, $startid) = @_;

  my $comments_map = {};
  my $do_login = 1;

  $LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
  $LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
  $LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
  $LJ::Simple::UserAgent = $LJR::USER_AGENT;

  my $i = 0;
  my $remote_lj;

  while (1) {
    if ($do_login) {
      $remote_lj = new LJ::Simple ({
        site => $server,
        user => $username,
        pass => $pass,
        pics  => 0,
        moods => 0,
        });
      return $err->("Can't login to remote site.", $LJ::Simple::error) unless defined $remote_lj;

      if (!$remote_lj->GenerateCookie()) {
        if (!$remote_lj->GenerateCookie()) {
          return $err->("Can't generate login cookie.", $LJ::Simple::error);
        }
      }

      $do_login = 0;
    }

    # do not process those which were processed once
    if ($comments_map->{$startid}) {
      $startid++;
      next;
    }

    my $res = $remote_lj->GetRawData(
      {"url" => "/export_comments.bml?get=comment_meta&startid=" . $startid}
      );

    if ($res && $res->{content}) {
      $cmode = "comment_meta";

      my $xdata = $res->{content};
      LJR::unicode::force_utf8(\$xdata);
      eval { LJ::text_out(\$xdata); };

      my $p1 = new XML::Parser (
        Handlers => {
          Start  => \&xmlh_comment_start,
          End    => \&xmlh_comment_end,
          Char   => \&xmlh_comment_char
        });

      $xml_maxid = 0;
      $xmlerrt = "";

      eval { $p1->parse($xdata); };
      if ($@) {
        if ($i < $LJR::NETWORK_RETRIES) {
          if ($xdata =~ /Login Required/) {
            $do_login = 1;
          }

          $i++;
          LJR::NETWORK_SLEEP;
          next;
        }
        else {
          $dumpxml->($xdata, $username);
          return $err->("Runtime error parsing XML (meta, $startid): ", $@);
        }
      }
      if ($xmlerrt) {
        $dumpxml->($xdata, $username);
        return $err->("Error parsing XML (meta, $startid): ", $xmlerrt);
      }

      # xml was processed successfully
      $comments_map->{$startid} = 1;

      cstatus_print ("prefetched $got_max_commentid (skipped $empty_num) of $xml_maxid comments");
      if ($got_max_commentid + $empty_num < $xml_maxid) {
        $empty_num++;
        $startid = $got_max_commentid + $empty_num;
        next;
      }
      else {
        $got_max_commentid = 0 unless $got_max_commentid;
        $ru = LJR::Distributed::set_cu_field($ru, "remote_meta_maxid", $got_max_commentid);
        return $err->($ru->{errtext}) if $ru->{err};
        return undef;
      }
    }
    else {
      if ($i < $LJR::NETWORK_RETRIES) {
        LJR::NETWORK_SLEEP; $i++; next;
      }
      else {
        return $err->("can't get comments metadata: " . $LJ::Simple::error);
      }
    }
  }
}

sub get_usermaps {
  my ($server, $username, $pass, $startid) = @_;

  $ru = LJR::Distributed::get_remote_server($server);
  return $err->($ru->{"errtext"}) if $ru->{"err"};
  $ru->{username} = $username;
  $ru->{pass} = $pass;
  $ru = LJR::Distributed::get_cached_user($ru);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  $got_max_commentid = $startid - 1;

  cstatus_print ("caching commented users.");

  my $e = get_usermaps_cycled($server, $username, $pass, $startid);
  return $err->($e->{errtext}) if $e->{err};
}

sub get_comments_cycled {
  my ($server, $username, $pass, $startid) = @_;

  my $comments_map = {};
  my $do_login = 1;

  $ru = LJR::Distributed::get_remote_server($server) unless $ru->{serverid};
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  $ru->{username} = $username;

  $ru = LJR::Distributed::get_cached_user($ru);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  $LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
  $LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
  $LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
  $LJ::Simple::UserAgent = $LJR::USER_AGENT;

  my $i = 0;
  my $h_counter;
  my $remote_lj;
  
  while (1) {
    if ($do_login) {
      $remote_lj = new LJ::Simple ({
        site => $server,
        user => $username,
        pass => $pass,
        pics  => 0,
        moods => 0,
        });
      return $err->("Can't login to remote site.", $LJ::Simple::error) unless defined $remote_lj;
  
      if (!$remote_lj->GenerateCookie()) {
        if (!$remote_lj->GenerateCookie()) {
          return $err->("Can't generate login cookie.", $LJ::Simple::error);
        }
      }

      $do_login = 0;
    }

    # do not process those which were processed once
    if ($comments_map->{$startid}) {
      $startid++;
      next;
    }

    my $res = $remote_lj->GetRawData(
      {"url" => "/export_comments.bml?get=comment_body&props=1&startid=" . $startid}
      );

    if ($res && $res->{content}) {
      my $xdata = $res->{content};
      LJR::unicode::force_utf8(\$xdata);
      eval { LJ::text_out(\$xdata); };

      $cmode = "comment_body";
      my $p1 = new XML::Parser (
        Handlers => {
          Start  => \&xmlh_comment_start,
          End    => \&xmlh_comment_end,
          Char   => \&xmlh_comment_char
        });

      $xmlerrt = "";
      eval { $p1->parse($xdata); };
      if ($@) {
        if ($i < $LJR::NETWORK_RETRIES) {
          if ($xdata =~ /Login Required/) {
            $do_login = 1;
          }

          $i++;
          LJR::NETWORK_SLEEP;
          next;
        }
        else {
          $dumpxml->($xdata, $username);
          return $err->("Runtime error parsing XML (body, $startid): ", $@);
        }
      }
      if ($xmlerrt) {
        $dumpxml->($xdata, $username);
        return $err->("Error parsing XML (body, $startid): ", $xmlerrt);
      }

      # remember last cached comment number (which is equal to its id)
      $ru = LJR::Distributed::set_cu_field(
        $ru, "cached_comments_maxid",
        $ru->{cached_comments_maxid});
      return $err->($ru->{errtext}) if $ru->{err};

      # xml was processed successfully
      $comments_map->{$startid} = 1;

      cstatus_print ("getting comments. last id: $ru->{cached_comments_maxid}, skipping: $empty_num, just walked: $startid, max: $ru->{remote_meta_maxid}");
      if ($ru->{cached_comments_maxid} + $empty_num < $ru->{remote_meta_maxid}) {
        if ($empty_num > 0) {
          $empty_num =
            (POSIX::floor($ru->{cached_comments_maxid} / 100) + 1) * 100 -
            $ru->{cached_comments_maxid} +
            $h_counter * 100;

          $h_counter++;
        }
        else {
          $empty_num++;
          $h_counter = 0;
        }

        $startid = $ru->{cached_comments_maxid} + $empty_num;
        next;
      }
      else {
        return undef;
      }
    }
    else {
      if ($i < $LJR::NETWORK_RETRIES) {
        LJR::NETWORK_SLEEP; $i++; next;
      }
      else {
        return $err->("can't get comments: " . $LJ::Simple::error);
      }
    }
  }
}

sub get_comments {
  my ($server, $username, $pass, $startid) = @_;

  cstatus_print ("caching comments");

  LJ::disconnect_dbs();

  $soft_cached_keyworded_pics = {};
  $soft_cached_default_pics = {};
  $posters_without_names = {};

  $ru = LJR::Distributed::get_remote_server($server);
  return $err->($ru->{"errtext"}) if $ru->{"err"};
  $ru->{username} = $username;
  $ru->{pass} = $pass;
  $ru = LJR::Distributed::get_cached_user($ru);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  my $e; # for possible errors

  $ru = LJR::Distributed::get_cu_field($ru, "cached_comments_maxid");
  $ru->{cached_comments_maxid} = 0 if not defined $ru->{cached_comments_maxid};

  # don't want to download cached comments again
  $startid = $ru->{cached_comments_maxid} + 1
    if $ru->{cached_comments_maxid} > $startid;

  $ru = LJR::Distributed::get_cu_field($ru, "remote_meta_maxid");
  $ru->{remote_meta_maxid} = 0 if not defined $ru->{remote_meta_maxid};

  # try to minimize possible further delays
  $got_max_commentid = $ru->{remote_meta_maxid};
  $e = get_usermaps_cycled($server, $username, $pass, $got_max_commentid + 1);
  return $err->($e->{errtext}) if $e->{err};

  # get remote comments and cache them
  $got_max_commentid = $startid - 1;
  $e = get_comments_cycled($server, $username, $pass, $startid);
  return $err->($e->{errtext}) if $e->{err};

  $soft_cached_keyworded_pics = {};
  $soft_cached_default_pics = {};
  $posters_without_names = {};

  return undef;
}

sub create_imported_comments {
  my ($remote_site, $remote_user, $local_user) = @_;

  LJ::disconnect_dbs();

  my $ru = LJR::Distributed::get_remote_server($remote_site);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  $ru->{username} = $remote_user;
  $ru = LJR::Distributed::get_cached_user($ru);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  cstatus_print("creating comments.");

  $ru = LJR::Distributed::create_imported_comments($ru, $local_user);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  return undef;
}


return 1;