ljr/local/bin/ljrimport/ijournal.pl

#!/usr/bin/perl -w

use strict;
use Simple; # corrected LJ::Simple
use POSIX;
use XML::Parser;
use Unicode::String;
use Unicode::MapUTF8 qw(to_utf8 from_utf8 utf8_supported_charset);

require "$ENV{'LJHOME'}/cgi-bin/ljlib.pl";
require "ljr-defaults.pl";
require "ljr-links.pl";
require LJR::Distributed;
require LJR::unicode;
require LWPx::ParanoidAgent;
require "ipics.pl";
require "$ENV{'LJHOME'}/cgi-bin/ljprotocol.pl";

my $DEBUG=1;

# shared variables (between flat and xml mode) 
my $ru; # remote user
my %rfg=(); # remote friend groups
my ($ritem_id, $ranum, $rhtml_id); # remote entry ids
my $local_u; # local user being imported into
my $flags; # ljprotocol.pl flags
my $do_overwrite; # overwrite entries

my $warns;

# flat mode parameters
my $REMOTE_MAX_GET = 50; # livejournal.com lets us download no more than 50 events for a given day

# XML mode functions and variables
my $xmlerrt;
my $xmlerr = sub {
  my $expat = shift;

  my $cstack = "\ncallstack:";
  my $i = 0;
  while ( 1 ) {
    my $tfunc = (caller($i))[3];
    if ($tfunc && $tfunc ne "") {
      if ($tfunc !~ /\_\_ANON\_\_/) {
        $cstack .= " " . $tfunc;
      }
      $i = $i + 1;
    }
    else {
      last;
    }
  }

  $xmlerrt = join("\n", @_);
  $xmlerrt .= $cstack;
  $expat->finish();
};

my $dumptofile = sub {
  my ($fdata, $filename, $ext) = @_;

  my $t = `date +"%T"`;
  $t = substr($t, 0, length($t) - 1);
  open(my $outfile, ">$ENV{'LJHOME'}/logs/" . $filename .  "_" . $t . $ext);
  print $outfile "$fdata";
  close($outfile);
};

my %xentry = (); # current entry
my $ctext = ""; # current field value

my $root_tags = qr/^(livejournal)$/;
my $entry_tag = qr/^(entry)$/;
my $entry_data_tag = qr/^(itemid|eventtime|logtime|subject|event|security|allowmask|current_music|current_mood)$/;


# error handling
my $err = sub {
  my %res = ();

  $res{"err"} = 1;
  $res{"errtext"} = join ("\n", @_);
  
  if ($warns) {
    $res{"warns"} = $warns;
  }
  
  return \%res;
};

my $warn = sub {
  print "WARNING: " . join ("\n", @_);
  print "\n";
  
  if (!$warns || length($warns) < 255) {
    $warns .= $warns . join(" ", @_);
    if (substr($warns, 0, 244) ne $warns) {
      $warns = substr($warns, 0, 244) + "; and more";
    }
  }
};

sub jstatus_print {
  my $statustr = join("", @_);

  if ($DEBUG) {
    eval { LJR::Import::import_log($statustr); };

    if ($@) {
      print $@ . "\n";
      print $statustr . "\n";
    }
  }
}

# overwrite entry
sub check_overwrite {
  my ($local_u, $ru_id, $ritem_id, $overwrite) = @_;

  my $r = LJR::Distributed::get_local_itemid ($local_u, $ru_id, $ritem_id. "I");
  return $err->($r->{"errtext"}) if $r->{"err"};

  if ($r->{"itemid"} && $overwrite eq "1") {
    my %req = (
      'username' => $local_u->{'user'},
      'ownerid' => $local_u->{'user'},
      'clientversion' => $LJR::LJ_CLIENT,
      'ver' => $LJ::PROTOCOL_VER,
      'selecttype' => 'one',
      'itemid' => $r->{"itemid"},
      'getmenus' => 0,
      'lineendings' => "unix",
      'truncate' => 0,
      );

    my $err1;
    my $items = LJ::Protocol::do_request("getevents", \%req, \$err1, $flags);
    if ($err1) {
      my $errstr = LJ::Protocol::error_message($err1);
      return $err->($errstr);
    }

    my $h = @{$items->{events}}[0];
    LJ::delete_entry($local_u, $h->{itemid});

    $r = LJR::Distributed::get_local_itemid ($local_u, $ru_id, $ritem_id, "I");
    return $err->($r->{"errtext"}) if $r->{"err"};
  }
  elsif ($r->{"itemid"} && $overwrite eq "0") {
    return {"continue" => 0};
  }

  return {"continue" => 1};
}

# XML handlers
sub xmlh_entry_start() {
  my $expat = shift;
  my @params = @_;
  
  if ($params[0] =~ /$root_tags/) {
    # skip valid but meaningless tags
  }
  elsif ($params[0] =~ /$entry_tag/) {
    # we're starting to process new entry
    shift @params;

    %xentry = ();
  }
  elsif ($params[0] =~ /$entry_data_tag/) {
    $ctext = "";
  }
  else {
    return $xmlerr->($expat,
      "Unknown XML-structure: " . join (" ", @params),
      "at line " . $expat->current_line);
  }
}
sub xmlh_entry_end() {
  my $expat = shift;
  my @params = @_;

  if ($params[0] =~ /$root_tags/) {
    # almost finished
  }
  elsif ($params[0] =~ /$entry_tag/) {
    my $xe = xml_create_entry(\%xentry);
    return $xmlerr->($expat, "xml_create_entry: " . $xe->{errtext}) if $xe && $xe->{err};
  }
  elsif ($params[0] =~ /$entry_data_tag/) {
    $xentry{$params[0]} = $ctext;
#    print $params[0] . " => " . $ctext . "\n";
  }
  else {
    return $xmlerr->($expat,
      "Unknown tag: " . join (" ", @params),
      "at line " . $expat->current_line
      );
  }
}
sub xmlh_entry_char() {
  my $expat = shift;
  my $tt = join("",  @_);
  $ctext = $ctext . $tt;
}

# should be called after populating shared variables (see section above)
sub xml_create_entry {
  my ($xentry) = @_;

  return $err->("XML import: can't extract remote itemid.") unless $xentry->{"itemid"};
  $ritem_id = int($xentry->{"itemid"} / 256); # export.bml returns html_id instead of item_id

  my $is_gated = LJR::Distributed::get_local_itemid($local_u, $ru->{'ru_id'}, $ritem_id, "E");
  return $err->($is_gated->{"errtext"}) if $is_gated->{"err"};
  return {"err" => 0} if $is_gated->{'itemid'};
  
  my $r = check_overwrite($local_u, $ru->{'ru_id'}, $ritem_id, $do_overwrite);
  return $err->($r->{"errtext"}) if $r->{"err"};
  return unless $r->{"continue"};

  my ($min,$hour,$mday,$mon,$year);

  if ($xentry->{"eventtime"} =~ /(\d\d\d\d)\-(\d\d)\-(\d\d)\ (\d\d)\:(\d\d)/o) {
    $year = $1;
    $mon = $2;
    $mday = $3;
    $hour = $4;
    $min = $5;
  }
  else {
    return $err->("XML import: can't extract eventtime. remote itemid = " . $ritem_id);
  }

  my $moodid;
  if ($xentry->{"current_mood"}) {
    $moodid = LJ::mood_id($xentry->{"current_mood"});
  }

  LJR::Links::make_ljr_hrefs(
    LJR::Links::get_server_url($ru->{"servername"}, "base"),
    $ru->{"servername"}, \$xentry->{"event"}
    );
            
#  LJR::unicode::utf8ize(\$xentry->{"event"});
#  LJR::unicode::utf8ize(\$xentry->{"subject"});
#  LJR::unicode::utf8ize(\$xentry->{"current_mood"});
#  LJR::unicode::utf8ize(\$xentry->{"current_music"});
  
  # LJ now exports lj-polls (previously
  # they exported only links to polls)
  $xentry->{'event'} =~ s/<lj-poll>.+<\/lj-poll>//sog;

  my %req = (
    'mode' => 'postevent',
    'ljr-import' => 1,
    'ver' => $LJ::PROTOCOL_VER,
    'clientversion' => $LJR::LJ_CLIENT,
    'user' => $local_u->{'user'},
    'username' => $local_u->{'user'},
    'usejournal' => $local_u->{'user'},
    'getmenus' => 0,
    'lineendings' => "unix",
    'event' => $xentry->{"event"},
    'subject' => $xentry->{"subject"},
    'year' => $year,
    'mon' => $mon,
    'day' => $mday,
    'hour' => $hour,
    'min' => $min,
    'props' => {
      'current_moodid' => $moodid,
      'current_mood' => $xentry->{"current_mood"},
      'current_music' => $xentry->{"current_music"},
      'opt_preformatted' => 0,
      'opt_nocomments' => 0,
      'taglist' => "",
      'picture_keyword' => "",
      'opt_noemail' => 0,
      'unknown8bit' => 0,
      'opt_backdated' => 1,
    },
  );

  if ($xentry->{"security"} eq "public" || $xentry->{"security"} eq "private") {
    $req{'security'} = $xentry->{"security"};
    $req{'allowmask'} = 0;
  }
  elsif ($xentry->{"security"} eq "usemask" && $xentry->{"allowmask"} == 1) {
    $req{'security'} = 'usemask';
    $req{'allowmask'} = 1;
  }
  else {
    $req{'security'} = 'usemask';

    my @groups = ();
    foreach my $grp_id (keys %rfg) {
      if ($xentry->{"allowmask"}+0 & 1 << $grp_id) {
        push @groups, $rfg{$grp_id}->{name};
      }
    }

    my $mask = 0;
    while (my $grpname = shift @groups) {
      my $group = LJ::get_friend_group($local_u, {'name' => $grpname});
      if ($group) {
        $mask = $mask | (1 << $group->{groupnum});
      }
    }
    $req{'allowmask'} = $mask;
  }
  
  my %res = ();
  LJ::do_request(\%req, \%res, $flags);
  if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Missing required argument") {
    $warn->($res{"errmsg"} . " while processing " . $xentry->{"eventtime"});
    return;
  }
  if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Post too large") {
    $dumptofile->($req{'event'}, "large_" . $local_u->{'user'}, ".raw");
  }

  return $err->($xentry->{"eventtime"} . ": " . $res{"errmsg"}) unless $res{"success"} eq "OK";

  $r = LJR::Distributed::store_remote_itemid(
    $local_u,
    $res{"itemid"},
    $ru->{ru_id},
    $ritem_id,
    $xentry->{"itemid"});
  return $err->($xentry->{"eventtime"} . ": " . $r->{"errtext"}) if $r->{"err"};
  
  return {"err" => 0};
}

# do the actual import
sub import_journal {
  my (
    $throttle_speed,
    $remote_site, $remote_protocol, $remote_user, $remote_pass, $remote_shared_journal,
    $local_user, $local_shared_journal, $overwrite
  ) = @_;

  $do_overwrite = $overwrite;
  LJ::disconnect_dbs(); # force reconnection to the database

  if ($remote_shared_journal eq "") {
    $remote_shared_journal = undef;
  }
  if ($local_shared_journal eq "") {
    $local_shared_journal = undef;
  }

  my %gdc_hr = ();
  my %req = ();
  my %lfg = ();
  my %res = ();

  if ($remote_protocol ne "flat" && $remote_protocol ne "xml") {
    return $err->("Unsupported remote protocol $remote_protocol.");
  }

  $LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
  $LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
  $LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
  $LJ::Simple::UserAgent = $LJR::USER_AGENT;

  # login to the remote site
  my $remote_lj = new LJ::Simple ({
      site => $remote_site,
      user => $remote_user,
      pass => $remote_pass,
      pics  => 0,
      moods => 0,
      });
  if (! defined $remote_lj) {
    return $err->("Can't login to remote site.", $LJ::Simple::error);
  }

  if (!$remote_lj->GenerateCookie()) {
    if (!$remote_lj->GenerateCookie()) {
      return $err->("Can't generate login cookie.", $LJ::Simple::error);
    }
  }

  # since we're able to login with supplied credentials --
  # get and/or cache remote server and remote user ident
  $ru = LJR::Distributed::get_remote_server($remote_site);
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  # try to get userid
  my $idres;
  my $i1 = 0;
  while(1) {
    my $ua = LWPx::ParanoidAgent->new(timeout => 60);
    $ua->agent($LJR::USER_AGENT);
    # TODO: parameterize allpics.bml
    my $url = $ru->{"servername"} . "/users/" . $remote_user . "/info/" ;
    $idres = $ua->get($url);
    
    if (!($idres && ($idres->is_success || $idres->code == 403)) && $i1 < $LJR::NETWORK_RETRIES) {
      my $txt;
      #foreach my $k (keys %$idres) {
      #   $txt .= $k . "->(" . $idres->{$k} ."), ";
      #}
###_content->(500 DNS lookup timeout), _rc->(500), _headers->(HTTP::Headers=HASH(0x2d2ec70)), _msg->(DNS lookup timeout), _request->(HTTP::Request=HASH(0x2c61ac0)),

      $txt .= "_msg->" . $idres->{'_msg'} . ", ";
      foreach my $k (keys %{$idres->{'_headers'}}) {
          $txt .= "\n" . $k . ": " . $idres->{'_headers'}->{$k} ;
      }
      print STDERR "*** $url  $txt\n";

      LJR::NETWORK_SLEEP(); $i1++; next;
    }
    else { last; }
  }
  
  if (!($idres && ($idres->is_success || $idres->code == 403))) {
    return $err->("LWPx: Can't get remote user id: $remote_user\n");
  }
  if ($idres->content && $idres->content =~ /\<b\>$remote_user\<\/b\>\<\/a\>\ \((\d+)\)/s) {
    $ru->{"userid"} = $1;
  }

  $ru->{"username"} = $remote_user;
  $ru = LJR::Distributed::get_cached_user($ru); # populates $ru->{ru_id}
  return $err->($ru->{"errtext"}) if $ru->{"err"};

  # get local user object for user being imported into
  $local_u = LJ::load_user($local_user, 1);
  return $err->("Can't load local user $local_user.") unless $local_u;

  $ru = LJR::Distributed::remote_local_assoc($ru, $local_u);
  return $err->("error while getting remote-local association: " . $ru->{errtext})
    if $ru->{err};

  jstatus_print ("getting userpics");
  my $e = import_pics(
    $ru->{servername},
    $ru->{username},
    $remote_pass,
    $local_user,
    "", 0);
  return $err->("Can't import " . $ru->{username} . ": " . $e->{errtext})
    if $e->{err};

  # clear duplicate protection
  LJ::set_userprop($local_u, "dupsig_post", undef);

  # needed everywhere
  $flags = {
    'u' => $local_u,
    'noauth' => 1,
    'BMAX_EVENT' => 150000,
    'CMAX_EVENT' => 150000,
    'no-cache' => 1,
    'omit_underscore_check' => 1,
    };

  %req = ( 'mode' => 'login',
           'ver' => $LJ::PROTOCOL_VER,
           'clientversion' => $LJR::LJ_CLIENT,
           'user' => $local_u->{'user'},
           'getmenus' => 0,
         );
  %res = ();
  LJ::do_request(\%req, \%res, $flags);
  return $err->($res{'errmsg'}) unless $res{'success'} eq 'OK';

  jstatus_print ("getting friend groups");

  # get remote and local friend groups, mix them up, update on local server
  if (! defined $remote_lj->GetFriendGroups(\%rfg)) {
    return $err->("Failed to get groups on the remote site.", $LJ::Simple::error);
  }

  LJ::Protocol::do_request(
    {
      'mode' => 'getfriendgroups',
      'user' => $local_u->{'user'}, 
      'ver'  => $LJ::PROTOCOL_VER,
      'clientversion' => $LJR::LJ_CLIENT,
      'includegroups' => 1,
      'getmenus' => 0,
    },
    \%res,
    $flags
    );
  if (! $res{'success'} eq "OK") {
    return $err->("Unable to get local user" . $local_u->{'user'} . "groups",
      $res{'success'} . ":" . $res{'errmsg'});
  }

  # convert it to LJ::Simple hash
  while((my $k, my $v) = each %res) {
    $k=~/^frgrp_([0-9]+)_(.*)$/o || next;
    my ($id, $name) = ($1, $2);
    if (!exists $lfg{$id}) {
      $lfg{$id}={
        id  => $id,
        public  => 0,
      };
    }
    ($name eq "sortorder") && ($name="sort");
    $lfg{$id}->{$name}=$v;
  }

  # add nonexisting remote groups (identified by name) to local server
  foreach my $grp_id (keys %rfg) {
    my $e = 0;
    foreach my $lg (values %lfg) {
      if ($lg->{name} eq $rfg{$grp_id}->{name}) {
        $e = 1;
      }
      if ($lg->{name} =~ /default view/i) {
        $e = 1;
      }
    }

    if (!$e) {
      my $egroup = 1;
      foreach my $cgroup (sort { $a <=> $b } keys %lfg) {
        if ($egroup == $cgroup) {
          $egroup++;
        }
      }
      if ($egroup < 31) {
        $lfg{$egroup} = $rfg{$grp_id};
      }
    }
  }

  # create local friend groups (existing + copied)
  my $i = 0;
  %req = (
    'mode' => "editfriendgroups",
    'user' => $local_u->{'user'},
    'clientversion' => $LJR::LJ_CLIENT,
    'ver' => $LJ::PROTOCOL_VER,
    );
  
  # convert LJ::Simple hash back to ljprotocol hash
  foreach my $grpid (keys %lfg) {
    if ($grpid > 0 && $grpid < 31) {

      my $pname = "efg_set_" . $grpid . "_name";
      $req{$pname} = $lfg{$grpid}->{name};
      $i++;
    }
  }

  # do the actual request
  LJ::do_request(\%req, \%res, $flags);
  if (! $res{'success'} eq "OK") {
    return $err->(
      "Unable to update local user" . $local_u->{'user'} . "groups",
      $res{'success'} . ":" . $res{'errmsg'}
      );
  }

  # get remote days with entries
  if (! defined $remote_lj->GetDayCounts(\%gdc_hr, undef)) {
    return $err->("can't get day counts: ", $LJ::Simple::error);
  }

  # import entries by means of export.bml (XML format)
  if ($remote_protocol eq "xml") {
    my $mydc = {};

    foreach (sort {$a<=>$b} keys %gdc_hr) {
      my $timestamp = $_;

      my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
        localtime($timestamp);

      $mon++;
      $year = $year + 1900;

      $mydc->{$year}->{$mon} = 0;
      $mydc->{$year}->{$mon} = $mydc->{$year}->{$mon} + $gdc_hr{$timestamp};
    }
    
    foreach (sort {$a <=> $b} keys %{$mydc}) {
      my $y = $_;

      foreach (sort {$a <=> $b} keys %{$mydc->{$y}}) {
        jstatus_print ("getting XML data and creating local entries for " . $_ . "/" . $y);
        
        my $do_login = 0;
  
        while (1) {
          if ($do_login) {
            $remote_lj = new LJ::Simple ({
              site => $remote_site,
              user => $remote_user,
              pass => $remote_pass,
              pics  => 0,
              moods => 0,
            });
            if (! defined $remote_lj) {
              return $err->("Can't login to remote site.", $LJ::Simple::error);
            }
            if (!$remote_lj->GenerateCookie()) {
              if (!$remote_lj->GenerateCookie()) {
                return $err->("Can't generate login cookie.", $LJ::Simple::error);
              }
            }
          }
  
          my $res = $remote_lj->GetRawData({
            "url" => "/export_do.bml",
            "post-data" => {
              "authas" => $remote_user, # "nit",
              "format" => "xml",
              "encid" => 2, # utf-8; for full listing see htdocs/export.bml
              "header" => 1,
              "year" => $y,
              "month" => $_,
              "field_itemid" => 1,
              "field_eventtime" => 1,
              "field_logtime" => 1,
              "field_subject" => 1,
              "field_event" => 1,
              "field_security" => 1,
              "field_allowmask" => 1,
              "field_currents" => 1,
            }});

          if ($res && $res->{content}) {
            my $xdata = $res->{content};
            LJR::unicode::force_utf8(\$xdata);

            my $p1 = new XML::Parser (
              Handlers => {
                Start  => \&xmlh_entry_start,
                End    => \&xmlh_entry_end,
                Char   => \&xmlh_entry_char
              });
            
            eval { $p1->parse($xdata); };
            if ($@) {
              if ($i < $LJR::NETWORK_RETRIES) {
                if ($@ =~ /not\ well\-formed\ \(invalid\ token\)/) {
#                  $xdata <?xml version="1.0" encoding='windows-1251'?>
                }
    
                if ($xdata =~ /Login Required/) {
                  $do_login = 1;
                }

                LJR::NETWORK_SLEEP(); $i++; next;
              }
              else {
                $dumptofile->($xdata, "err_" . $remote_user, ".xml");
                return $err->("Runtime error while parsing XML data: ", $@);
              }
            }
            
            if ($xmlerrt) {
              $dumptofile->($xdata, "err_" . $remote_user, ".xml");
              return $err->("Error while parsing XML data: ", $xmlerrt);
            }

            last;
          }
          else {
            return $err->("Can't get XML data..");
          }
        }
      }
    }
  }
  
  # import entries by means of flat protocol
  if ($remote_protocol eq "flat") {
    # process them, day by day, sleeping a little
    foreach (sort {$a<=>$b} keys %gdc_hr) {
      my $timestamp = $_;

      # download all the entries for a day
      if ($gdc_hr{$timestamp} < $REMOTE_MAX_GET) {
        jstatus_print (
          "getting remote and creating local entries for " .
          strftime ("%a %b %e %Y", localtime($timestamp))
          );

        my %r_entries=(); # remote entries

        if (! defined $remote_lj->GetEntries(\%r_entries,$remote_shared_journal,"day",($timestamp))) {
          if ($LJ::Simple::error =~ "Cannot display this post") {
      $warn->(strftime ("%a %b %e %Y", localtime($timestamp)) . ":" . $LJ::Simple::error);
      next;
    }
    return $err->("can't get remote entries: " . strftime ("%a %b %e %Y", localtime($timestamp)) . ": ",
            $LJ::Simple::error);
        }

        my $rkey=undef;
        my $rentry=undef;
        my $r;

        ENTRIES: while (($rkey, $rentry) = each(%r_entries)) {
          ($ritem_id, $ranum, $rhtml_id) = $remote_lj->GetItemId($rentry);
          my $tevent = $remote_lj->GetEntry($rentry);
          
          my $is_gated = LJR::Distributed::get_local_itemid($local_u, $ru->{'ru_id'}, $ritem_id, "E");
          return $err->($is_gated->{"errtext"}) if $is_gated->{"err"};
          next ENTRIES if $is_gated->{'itemid'};

          $r = check_overwrite($local_u, $ru->{'ru_id'}, $ritem_id, $do_overwrite);
          return $err->($r->{"errtext"}) if $r->{"err"};

          next ENTRIES unless $r->{"continue"};

          my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
            localtime($remote_lj->GetDate($rentry));

          $mon++;
          $year = $year + 1900;

          LJR::Links::make_ljr_hrefs(
            LJR::Links::get_server_url($ru->{"servername"}, "base"),
            $ru->{"servername"},
            \$tevent
            );
        
          my $tsubject = $remote_lj->GetSubject($rentry);
          my $tcurrent_mood = $remote_lj->Getprop_current_mood($rentry);
          my $tcurrent_music = $remote_lj->Getprop_current_music($rentry);
          my $ttaglist = $remote_lj->Getprop_taglist($rentry);
    $ttaglist = LJ::trim($ttaglist);
    
          my $tpicture_keyword = $remote_lj->Getprop_picture_keyword($rentry);

#          LJR::unicode::utf8ize(\$tevent);
#          LJR::unicode::utf8ize(\$tsubject);
#          LJR::unicode::utf8ize(\$tcurrent_mood);
#          LJR::unicode::utf8ize(\$tcurrent_music);
#          LJR::unicode::utf8ize(\$ttaglist);
#          LJR::unicode::utf8ize(\$tpicture_keyword);

          %req = ( 'mode' => 'postevent',
                   'ljr-import' => 1,
                   'ver' => $LJ::PROTOCOL_VER,
                   'clientversion' => $LJR::LJ_CLIENT,
                   'user' => $local_u->{'user'},
                   'username' => $local_u->{'user'},
                   'usejournal' => $local_u->{'user'},
                   'getmenus' => 0,
                   'lineendings' => "unix",
                   'event' => $tevent,
                   'subject' => $tsubject,
                   'year' => $year,
                   'mon' => $mon,
                   'day' => $mday,
                   'hour' => $hour,
                   'min' => $min,
                   'props' => {
                     'current_moodid' => $rentry->{prop_current_moodid},
                     'current_mood' => $tcurrent_mood,
                     'current_music' => $tcurrent_music,
                     'opt_preformatted' => $remote_lj->Getprop_preformatted($rentry),
                     'opt_nocomments' => $remote_lj->Getprop_nocomments($rentry),
                     'taglist' => $ttaglist,
                     'picture_keyword' => $tpicture_keyword,
                     'opt_noemail' => $remote_lj->Getprop_noemail($rentry),
                     'unknown8bit' => $remote_lj->Getprop_unknown8bit($rentry),
                     'opt_backdated' => 1,
                   },
                 );

          my @r_protection = $remote_lj->GetProtect($rentry);
          if ($r_protection[0] eq "public" || $r_protection[0] eq "private") {
            $req{'security'} = $r_protection[0];
            $req{'allowmask'} = 0;
          }
          elsif ($r_protection[0] eq "friends") {
            $req{'security'} = 'usemask';
            $req{'allowmask'} = 1;
          }
          elsif ($r_protection[0] eq "groups") {
            $req{'security'} = 'usemask';
            shift @r_protection;

            my $mask=0;
            while (my $grpname = shift @r_protection) {
              my $group = LJ::get_friend_group($local_u, {'name' => $grpname});
              $mask = $mask | (1 << $group->{groupnum});
            }
            $req{'allowmask'} = $mask;
          }

          %res = ();
          LJ::do_request(\%req, \%res, $flags);
          if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Post too large") {
            $dumptofile->($req{'event'}, "large_" . $local_u->{'user'}, ".raw");
          }
    if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Invalid text encoding") {
      $warn->($res{"errmsg"});
      next;
    }

    if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Invalid or malformed tag list") {
      return $err->($res{"errmsg"} . ": [$ttaglist]");
    }

    return $err->($res{'errmsg'}) unless $res{'success'} eq 'OK';
          
    $r = LJR::Distributed::store_remote_itemid(
            $local_u,
            $res{"itemid"},
            $ru->{ru_id},
            $ritem_id,
            $rhtml_id);
          return $err->($r->{"errtext"}) if $r->{"err"};
        }

        sleep($throttle_speed);
      }
      else {
        $warn->("Too much entries for a day. " . $local_u->{'user'} . " " .
          strftime ("%a %b %e %Y", localtime($timestamp))
          );
      }
    } # process them day by day
  }

  if ($warns) {
    my %warns = ('warns' => $warns);
    return \%warns;
  }
  else {
    return undef;
  }
}


return 1;