zlax/ljr
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

init

Browse Source
This commit is contained in:
ivan
2019-02-06 00:49:12 +03:00
commit 8dbb1bb605
4796 changed files with 506072 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4596
local/bin/ljrimport/Simple.pm Executable file
View File

File diff suppressed because it is too large Load Diff

647
local/bin/ljrimport/icomments.pl Executable file
View File

@@ -0,0 +1,647 @@
#!/usr/bin/perl -w
use strict;
use Simple; # corrected LJ::Simple
use XML::Parser;
use POSIX;
require "$ENV{'LJHOME'}/cgi-bin/ljlib.pl";
require "ljr-defaults.pl";
require "ljr-links.pl";
require LJR::Distributed;
require LJR::unicode;
require "$ENV{'LJHOME'}/cgi-bin/talklib.pl";
require "ipics.pl";
my $err = sub {
my %res = ();
$res{"err"} = 1;
$res{"errtext"} = join ("\n", @_);
return \%res;
};
my $xmlerrt;
my $xmlerr = sub {
my $expat = shift;
my $cstack = "\ncallstack:";
my $i = 0;
while ( 1 ) {
my $tfunc = (caller($i))[3];
if ($tfunc && $tfunc ne "") {
if ($tfunc !~ /\_\_ANON\_\_/) {
$cstack .= " " . $tfunc;
}
$i = $i + 1;
}
else {
last;
}
}
$xmlerrt = join("\n", @_);
$xmlerrt .= $cstack;
$expat->finish();
};
my $dumpxml = sub {
my ($xdata, $username) = @_;
my $t = `date +"%T"`;
$t = substr($t, 0, length($t) - 1);
open(my $outfile, ">$ENV{'LJHOME'}/logs/err_" . $username . "_" . $t . ".xml");
print $outfile "$xdata";
close($outfile);
};
# kind of global variables
my $DEBUG=1;
my $got_max_commentid;
my $empty_num = 0;
my $ru; # current remote user (the one being cached and imported)
my $cmode;
my $xml_maxid = 0;
my $soft_cached_default_pics;
my $soft_cached_keyworded_pics;
my $posters_without_names; # lj bug workaround: http://rt.livejournal.org/Ticket/Display.html?id=762
sub cstatus_print {
my $statustr = join("", @_);
eval { LJR::Import::import_log($statustr); };
if ($@) {
print $@ . "\n";
}
if ($DEBUG) {
eval { LJR::Import::log_print($statustr); };
if ($@) {
print $statustr . "\n";
}
}
}
# comment processing
my $skip_tags = qr/^(livejournal|comments|usermaps|nextid)$/;
my $maxid = qr/^(maxid)$/;
my $comment_tag = qr/^(comment)$/;
my $usermap_tag = qr/^(usermap)$/;
my $data_tags = qr/^(subject|body|date)$/;
my $prop_tag = qr/^(property)$/;
LJ::load_props("talk");
return $err->("Can't load talkprops.") unless $LJ::CACHE_PROP{talk};
my @tprops;
foreach my $k (keys %{$LJ::CACHE_PROP{talk}}) {
push(@tprops, $k);
}
my $tprops = join("|", @tprops);
my $known_props = qr/^($tprops)$/;
my %comment = (); # current comment structure
my $cid = ""; # current tag (inside comment tag)
my $ctext = ""; # current value of $cid
# export_comments.bml xml handling routines
sub xmlh_comment_start() {
my $expat = shift;
my @params = @_;
if ($params[0] =~ /$skip_tags/) {
# skip valid but meaningless tags
}
elsif ($cmode eq "comment_meta" && $params[0] =~ /$usermap_tag/) {
shift @params; # skip "usermap"
my %usermap = ();
%usermap = @params;
if ($usermap{id} && $usermap{user}) {
my $r = {
serverid => $ru->{serverid},
userid => $usermap{id},
username => $usermap{user},
};
$r = LJR::Distributed::get_cached_user($r);
return $xmlerr->($expat, $r->{errtext}) if $r->{err};
$r = LJR::Distributed::get_imported_user($r);
return $xmlerr->($expat, $r->{errtext}) if $r->{err};
$r = LJR::Distributed::set_cu_field($r, "local_commenterid", $r->{commenterid});
return $xmlerr->($expat, $r->{errtext}) if $r->{err};
if (!$soft_cached_default_pics->{$usermap{id}}) {
my $iu = LJ::load_userid ($r->{local_commenterid});
# cstatus_print(
# "caching default userpic for " . $r->{username} .
# " (" . $r->{local_commenterid} . ":" . $iu->{user} . ")"
# );
my $e = import_pics(
$ru->{servername},
$usermap{user},
"",
$iu->{user},
"", 1);
return $xmlerr->($expat, "importing default userpic for [" . $usermap{user} . "]:", $e->{errtext})
if $e->{err};
$soft_cached_default_pics->{$usermap{id}} = 1;
}
}
elsif ($usermap{id} && !$usermap{user}) {
$posters_without_names->{$usermap{id}} = 1;
}
else {
return $xmlerr->($expat,
"Unknown XML-structure: " . join (" ", @params),
"at line " . $expat->current_line);
}
}
elsif ($params[0] =~ /$comment_tag/) {
# we're starting to process new comment
shift @params;
%comment = ();
%comment = @params; # grab all comment attributes
}
elsif ($cmode eq "comment_body" && $params[0] =~ /$data_tags/) {
$cid = $params[0];
$ctext = "";
}
elsif ($cmode eq "comment_body" && $params[0] =~ /$prop_tag/) {
shift @params; # skip "property"
# skip "name" attribute name
if (shift @params && $params[0] =~ /$known_props/) {
$cid = $params[0];
$ctext = "";
}
}
elsif ($params[0] =~ /$maxid/) {
$ctext = "";
}
else {
return $xmlerr->($expat,
"Unknown XML-structure: " . join (" ", @params),
"at line " . $expat->current_line);
}
}
sub xmlh_comment_end() {
my $expat = shift;
my @params = @_;
if ($params[0] =~ /$skip_tags/) {
# almost finished
}
elsif ($cmode eq "comment_meta" && $params[0] =~ /$usermap_tag/) {
# nop
}
elsif ($params[0] =~ /$comment_tag/) {
if ($cmode eq "comment_body") {
# print $comment{"id"} . "\n";
# print "COMMENT\n";
# while ((my $k, my $v) = each(%comment)) {
# print $k . ":" . $v . "\n";
# }
# print "/COMMENT\n";
$comment{ru_id} = $ru->{ru_id};
if (
$comment{props} &&
$comment{props}->{"picture_keyword"} &&
$comment{posterid} &&
!$soft_cached_keyworded_pics->{$comment{posterid}}->{$comment{props}->{"picture_keyword"}}
) {
my $r = {
serverid => $ru->{serverid},
userid => $comment{posterid},
};
$r = LJR::Distributed::get_cached_user($r);
return $xmlerr->($expat, $r->{errtext}) if $r->{err};
$r = LJR::Distributed::get_imported_user($r);
return $xmlerr->($expat, $r->{errtext} . "(userid: " . $comment{'posterid'} . ")") if $r->{err};
$r = LJR::Distributed::get_cu_field($r, "local_commenterid");
return $xmlerr->($expat, $r->{errtext}) if $r->{err};
my $iu = LJ::load_userid ($r->{local_commenterid});
#cstatus_print ("caching userpic " . $comment{props}->{"picture_keyword"} . " for " . $r->{username} . ":" . $iu->{user});
my $e = import_pics (
$ru->{servername},
$r->{username},
"",
$iu->{user},
$comment{props}->{"picture_keyword"},
0);
return $xmlerr->($expat, $e->{errtext}) if $e->{err};
$soft_cached_keyworded_pics->{$comment{posterid}}->{$comment{props}->{"picture_keyword"}} = 1;
}
LJR::Links::make_ljr_hrefs(
LJR::Links::get_server_url($ru->{"servername"}, "base"),
$ru->{"servername"}, \$comment{body}
);
if ($comment{'posterid'} && $posters_without_names->{$comment{'posterid'}}) {
$comment{'posterid'} = undef;
}
if (!$comment{'body'} && $comment{'state'} ne "D") {
$comment{'body'} = "LJR::Import warning: no comment body during import.";
}
my $c = LJR::Distributed::cache_comment (\%comment);
return $xmlerr->($expat, $c->{'errtext'}) if $c->{'err'};
if (!$ru->{cached_comments_maxid} ||
$comment{id} > $ru->{cached_comments_maxid}) {
$ru->{cached_comments_maxid} = $comment{id};
}
}
$got_max_commentid++;
$empty_num = 0;
}
elsif ($params[0] =~ /$data_tags/) {
$comment{$cid} = $ctext;
}
elsif ($params[0] =~ /$prop_tag/) {
$comment{props}->{$cid} = $ctext;
}
elsif ($params[0] =~ /$maxid/) {
$xml_maxid = $ctext;
if ($cmode eq "comment_body" && $xml_maxid > $ru->{remote_meta_maxid}) {
my $tmid = $got_max_commentid;
my $txid = $xml_maxid;
my $tempty = $empty_num;
$got_max_commentid = $ru->{remote_meta_maxid};
my $e = get_usermaps_cycled(
$ru->{servername},
$ru->{username},
$ru->{pass},
$got_max_commentid + 1);
return $xmlerr->($expat, $e->{errtext}) if $e->{err};
# restore comment_body xml-parsing mode
$xml_maxid = $txid;
$got_max_commentid = $tmid;
$empty_num = $tempty;
$cmode = "comment_body";
}
}
else {
return $xmlerr->($expat,
"Unknown tag: " . join (" ", @params),
"at line " . $expat->current_line
);
}
}
sub xmlh_comment_char() {
my $expat = shift;
my $tt = join("", @_);
$ctext = $ctext . $tt;
}
sub get_usermaps_cycled {
my ($server, $username, $pass, $startid) = @_;
my $comments_map = {};
my $do_login = 1;
$LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
$LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
$LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
$LJ::Simple::UserAgent = $LJR::USER_AGENT;
my $i = 0;
my $remote_lj;
while (1) {
if ($do_login) {
$remote_lj = new LJ::Simple ({
site => $server,
user => $username,
pass => $pass,
pics => 0,
moods => 0,
});
return $err->("Can't login to remote site.", $LJ::Simple::error) unless defined $remote_lj;
if (!$remote_lj->GenerateCookie()) {
if (!$remote_lj->GenerateCookie()) {
return $err->("Can't generate login cookie.", $LJ::Simple::error);
}
}
$do_login = 0;
}
# do not process those which were processed once
if ($comments_map->{$startid}) {
$startid++;
next;
}
my $res = $remote_lj->GetRawData(
{"url" => "/export_comments.bml?get=comment_meta&startid=" . $startid}
);
if ($res && $res->{content}) {
$cmode = "comment_meta";
my $xdata = $res->{content};
LJR::unicode::force_utf8(\$xdata);
eval { LJ::text_out(\$xdata); };
my $p1 = new XML::Parser (
Handlers => {
Start => \&xmlh_comment_start,
End => \&xmlh_comment_end,
Char => \&xmlh_comment_char
});
$xml_maxid = 0;
$xmlerrt = "";
eval { $p1->parse($xdata); };
if ($@) {
if ($i < $LJR::NETWORK_RETRIES) {
if ($xdata =~ /Login Required/) {
$do_login = 1;
}
$i++;
LJR::NETWORK_SLEEP;
next;
}
else {
$dumpxml->($xdata, $username);
return $err->("Runtime error parsing XML (meta, $startid): ", $@);
}
}
if ($xmlerrt) {
$dumpxml->($xdata, $username);
return $err->("Error parsing XML (meta, $startid): ", $xmlerrt);
}
# xml was processed successfully
$comments_map->{$startid} = 1;
cstatus_print ("prefetched $got_max_commentid (skipped $empty_num) of $xml_maxid comments");
if ($got_max_commentid + $empty_num < $xml_maxid) {
$empty_num++;
$startid = $got_max_commentid + $empty_num;
next;
}
else {
$got_max_commentid = 0 unless $got_max_commentid;
$ru = LJR::Distributed::set_cu_field($ru, "remote_meta_maxid", $got_max_commentid);
return $err->($ru->{errtext}) if $ru->{err};
return undef;
}
}
else {
if ($i < $LJR::NETWORK_RETRIES) {
LJR::NETWORK_SLEEP; $i++; next;
}
else {
return $err->("can't get comments metadata: " . $LJ::Simple::error);
}
}
}
}
sub get_usermaps {
my ($server, $username, $pass, $startid) = @_;
$ru = LJR::Distributed::get_remote_server($server);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$ru->{username} = $username;
$ru->{pass} = $pass;
$ru = LJR::Distributed::get_cached_user($ru);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$got_max_commentid = $startid - 1;
cstatus_print ("caching commented users.");
my $e = get_usermaps_cycled($server, $username, $pass, $startid);
return $err->($e->{errtext}) if $e->{err};
}
sub get_comments_cycled {
my ($server, $username, $pass, $startid) = @_;
my $comments_map = {};
my $do_login = 1;
$ru = LJR::Distributed::get_remote_server($server) unless $ru->{serverid};
return $err->($ru->{"errtext"}) if $ru->{"err"};
$ru->{username} = $username;
$ru = LJR::Distributed::get_cached_user($ru);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
$LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
$LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
$LJ::Simple::UserAgent = $LJR::USER_AGENT;
my $i = 0;
my $h_counter;
my $remote_lj;
while (1) {
if ($do_login) {
$remote_lj = new LJ::Simple ({
site => $server,
user => $username,
pass => $pass,
pics => 0,
moods => 0,
});
return $err->("Can't login to remote site.", $LJ::Simple::error) unless defined $remote_lj;
if (!$remote_lj->GenerateCookie()) {
if (!$remote_lj->GenerateCookie()) {
return $err->("Can't generate login cookie.", $LJ::Simple::error);
}
}
$do_login = 0;
}
# do not process those which were processed once
if ($comments_map->{$startid}) {
$startid++;
next;
}
my $res = $remote_lj->GetRawData(
{"url" => "/export_comments.bml?get=comment_body&props=1&startid=" . $startid}
);
if ($res && $res->{content}) {
my $xdata = $res->{content};
LJR::unicode::force_utf8(\$xdata);
eval { LJ::text_out(\$xdata); };
$cmode = "comment_body";
my $p1 = new XML::Parser (
Handlers => {
Start => \&xmlh_comment_start,
End => \&xmlh_comment_end,
Char => \&xmlh_comment_char
});
$xmlerrt = "";
eval { $p1->parse($xdata); };
if ($@) {
if ($i < $LJR::NETWORK_RETRIES) {
if ($xdata =~ /Login Required/) {
$do_login = 1;
}
$i++;
LJR::NETWORK_SLEEP;
next;
}
else {
$dumpxml->($xdata, $username);
return $err->("Runtime error parsing XML (body, $startid): ", $@);
}
}
if ($xmlerrt) {
$dumpxml->($xdata, $username);
return $err->("Error parsing XML (body, $startid): ", $xmlerrt);
}
# remember last cached comment number (which is equal to its id)
$ru = LJR::Distributed::set_cu_field(
$ru, "cached_comments_maxid",
$ru->{cached_comments_maxid});
return $err->($ru->{errtext}) if $ru->{err};
# xml was processed successfully
$comments_map->{$startid} = 1;
cstatus_print ("getting comments. last id: $ru->{cached_comments_maxid}, skipping: $empty_num, just walked: $startid, max: $ru->{remote_meta_maxid}");
if ($ru->{cached_comments_maxid} + $empty_num < $ru->{remote_meta_maxid}) {
if ($empty_num > 0) {
$empty_num =
(POSIX::floor($ru->{cached_comments_maxid} / 100) + 1) * 100 -
$ru->{cached_comments_maxid} +
$h_counter * 100;
$h_counter++;
}
else {
$empty_num++;
$h_counter = 0;
}
$startid = $ru->{cached_comments_maxid} + $empty_num;
next;
}
else {
return undef;
}
}
else {
if ($i < $LJR::NETWORK_RETRIES) {
LJR::NETWORK_SLEEP; $i++; next;
}
else {
return $err->("can't get comments: " . $LJ::Simple::error);
}
}
}
}
sub get_comments {
my ($server, $username, $pass, $startid) = @_;
cstatus_print ("caching comments");
LJ::disconnect_dbs();
$soft_cached_keyworded_pics = {};
$soft_cached_default_pics = {};
$posters_without_names = {};
$ru = LJR::Distributed::get_remote_server($server);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$ru->{username} = $username;
$ru->{pass} = $pass;
$ru = LJR::Distributed::get_cached_user($ru);
return $err->($ru->{"errtext"}) if $ru->{"err"};
my $e; # for possible errors
$ru = LJR::Distributed::get_cu_field($ru, "cached_comments_maxid");
$ru->{cached_comments_maxid} = 0 if not defined $ru->{cached_comments_maxid};
# don't want to download cached comments again
$startid = $ru->{cached_comments_maxid} + 1
if $ru->{cached_comments_maxid} > $startid;
$ru = LJR::Distributed::get_cu_field($ru, "remote_meta_maxid");
$ru->{remote_meta_maxid} = 0 if not defined $ru->{remote_meta_maxid};
# try to minimize possible further delays
$got_max_commentid = $ru->{remote_meta_maxid};
$e = get_usermaps_cycled($server, $username, $pass, $got_max_commentid + 1);
return $err->($e->{errtext}) if $e->{err};
# get remote comments and cache them
$got_max_commentid = $startid - 1;
$e = get_comments_cycled($server, $username, $pass, $startid);
return $err->($e->{errtext}) if $e->{err};
$soft_cached_keyworded_pics = {};
$soft_cached_default_pics = {};
$posters_without_names = {};
return undef;
}
sub create_imported_comments {
my ($remote_site, $remote_user, $local_user) = @_;
LJ::disconnect_dbs();
my $ru = LJR::Distributed::get_remote_server($remote_site);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$ru->{username} = $remote_user;
$ru = LJR::Distributed::get_cached_user($ru);
return $err->($ru->{"errtext"}) if $ru->{"err"};
cstatus_print("creating comments.");
$ru = LJR::Distributed::create_imported_comments($ru, $local_user);
return $err->($ru->{"errtext"}) if $ru->{"err"};
return undef;
}
return 1;

839
local/bin/ljrimport/ijournal.pl Executable file
View File

@@ -0,0 +1,839 @@
#!/usr/bin/perl -w
use strict;
use Simple; # corrected LJ::Simple
use POSIX;
use XML::Parser;
use Unicode::String;
use Unicode::MapUTF8 qw(to_utf8 from_utf8 utf8_supported_charset);
require "$ENV{'LJHOME'}/cgi-bin/ljlib.pl";
require "ljr-defaults.pl";
require "ljr-links.pl";
require LJR::Distributed;
require LJR::unicode;
require LWPx::ParanoidAgent;
require "ipics.pl";
require "$ENV{'LJHOME'}/cgi-bin/ljprotocol.pl";
my $DEBUG=1;
# shared variables (between flat and xml mode)
my $ru; # remote user
my %rfg=(); # remote friend groups
my ($ritem_id, $ranum, $rhtml_id); # remote entry ids
my $local_u; # local user being imported into
my $flags; # ljprotocol.pl flags
my $do_overwrite; # overwrite entries
my $warns;
# flat mode parameters
my $REMOTE_MAX_GET = 50; # livejournal.com lets us download no more than 50 events for a given day
# XML mode functions and variables
my $xmlerrt;
my $xmlerr = sub {
my $expat = shift;
my $cstack = "\ncallstack:";
my $i = 0;
while ( 1 ) {
my $tfunc = (caller($i))[3];
if ($tfunc && $tfunc ne "") {
if ($tfunc !~ /\_\_ANON\_\_/) {
$cstack .= " " . $tfunc;
}
$i = $i + 1;
}
else {
last;
}
}
$xmlerrt = join("\n", @_);
$xmlerrt .= $cstack;
$expat->finish();
};
my $dumptofile = sub {
my ($fdata, $filename, $ext) = @_;
my $t = `date +"%T"`;
$t = substr($t, 0, length($t) - 1);
open(my $outfile, ">$ENV{'LJHOME'}/logs/" . $filename . "_" . $t . $ext);
print $outfile "$fdata";
close($outfile);
};
my %xentry = (); # current entry
my $ctext = ""; # current field value
my $root_tags = qr/^(livejournal)$/;
my $entry_tag = qr/^(entry)$/;
my $entry_data_tag = qr/^(itemid|eventtime|logtime|subject|event|security|allowmask|current_music|current_mood)$/;
# error handling
my $err = sub {
my %res = ();
$res{"err"} = 1;
$res{"errtext"} = join ("\n", @_);
if ($warns) {
$res{"warns"} = $warns;
}
return \%res;
};
my $warn = sub {
print "WARNING: " . join ("\n", @_);
print "\n";
if (!$warns || length($warns) < 255) {
$warns .= $warns . join(" ", @_);
if (substr($warns, 0, 244) ne $warns) {
$warns = substr($warns, 0, 244) + "; and more";
}
}
};
sub jstatus_print {
my $statustr = join("", @_);
if ($DEBUG) {
eval { LJR::Import::import_log($statustr); };
if ($@) {
print $@ . "\n";
print $statustr . "\n";
}
}
}
# overwrite entry
sub check_overwrite {
my ($local_u, $ru_id, $ritem_id, $overwrite) = @_;
my $r = LJR::Distributed::get_local_itemid ($local_u, $ru_id, $ritem_id. "I");
return $err->($r->{"errtext"}) if $r->{"err"};
if ($r->{"itemid"} && $overwrite eq "1") {
my %req = (
'username' => $local_u->{'user'},
'ownerid' => $local_u->{'user'},
'clientversion' => $LJR::LJ_CLIENT,
'ver' => $LJ::PROTOCOL_VER,
'selecttype' => 'one',
'itemid' => $r->{"itemid"},
'getmenus' => 0,
'lineendings' => "unix",
'truncate' => 0,
);
my $err1;
my $items = LJ::Protocol::do_request("getevents", \%req, \$err1, $flags);
if ($err1) {
my $errstr = LJ::Protocol::error_message($err1);
return $err->($errstr);
}
my $h = @{$items->{events}}[0];
LJ::delete_entry($local_u, $h->{itemid});
$r = LJR::Distributed::get_local_itemid ($local_u, $ru_id, $ritem_id, "I");
return $err->($r->{"errtext"}) if $r->{"err"};
}
elsif ($r->{"itemid"} && $overwrite eq "0") {
return {"continue" => 0};
}
return {"continue" => 1};
}
# XML handlers
sub xmlh_entry_start() {
my $expat = shift;
my @params = @_;
if ($params[0] =~ /$root_tags/) {
# skip valid but meaningless tags
}
elsif ($params[0] =~ /$entry_tag/) {
# we're starting to process new entry
shift @params;
%xentry = ();
}
elsif ($params[0] =~ /$entry_data_tag/) {
$ctext = "";
}
else {
return $xmlerr->($expat,
"Unknown XML-structure: " . join (" ", @params),
"at line " . $expat->current_line);
}
}
sub xmlh_entry_end() {
my $expat = shift;
my @params = @_;
if ($params[0] =~ /$root_tags/) {
# almost finished
}
elsif ($params[0] =~ /$entry_tag/) {
my $xe = xml_create_entry(\%xentry);
return $xmlerr->($expat, "xml_create_entry: " . $xe->{errtext}) if $xe && $xe->{err};
}
elsif ($params[0] =~ /$entry_data_tag/) {
$xentry{$params[0]} = $ctext;
# print $params[0] . " => " . $ctext . "\n";
}
else {
return $xmlerr->($expat,
"Unknown tag: " . join (" ", @params),
"at line " . $expat->current_line
);
}
}
sub xmlh_entry_char() {
my $expat = shift;
my $tt = join("", @_);
$ctext = $ctext . $tt;
}
# should be called after populating shared variables (see section above)
sub xml_create_entry {
my ($xentry) = @_;
return $err->("XML import: can't extract remote itemid.") unless $xentry->{"itemid"};
$ritem_id = int($xentry->{"itemid"} / 256); # export.bml returns html_id instead of item_id
my $is_gated = LJR::Distributed::get_local_itemid($local_u, $ru->{'ru_id'}, $ritem_id, "E");
return $err->($is_gated->{"errtext"}) if $is_gated->{"err"};
return {"err" => 0} if $is_gated->{'itemid'};
my $r = check_overwrite($local_u, $ru->{'ru_id'}, $ritem_id, $do_overwrite);
return $err->($r->{"errtext"}) if $r->{"err"};
return unless $r->{"continue"};
my ($min,$hour,$mday,$mon,$year);
if ($xentry->{"eventtime"} =~ /(\d\d\d\d)\-(\d\d)\-(\d\d)\ (\d\d)\:(\d\d)/o) {
$year = $1;
$mon = $2;
$mday = $3;
$hour = $4;
$min = $5;
}
else {
return $err->("XML import: can't extract eventtime. remote itemid = " . $ritem_id);
}
my $moodid;
if ($xentry->{"current_mood"}) {
$moodid = LJ::mood_id($xentry->{"current_mood"});
}
LJR::Links::make_ljr_hrefs(
LJR::Links::get_server_url($ru->{"servername"}, "base"),
$ru->{"servername"}, \$xentry->{"event"}
);
# LJR::unicode::utf8ize(\$xentry->{"event"});
# LJR::unicode::utf8ize(\$xentry->{"subject"});
# LJR::unicode::utf8ize(\$xentry->{"current_mood"});
# LJR::unicode::utf8ize(\$xentry->{"current_music"});
# LJ now exports lj-polls (previously
# they exported only links to polls)
$xentry->{'event'} =~ s/<lj-poll>.+<\/lj-poll>//sog;
my %req = (
'mode' => 'postevent',
'ljr-import' => 1,
'ver' => $LJ::PROTOCOL_VER,
'clientversion' => $LJR::LJ_CLIENT,
'user' => $local_u->{'user'},
'username' => $local_u->{'user'},
'usejournal' => $local_u->{'user'},
'getmenus' => 0,
'lineendings' => "unix",
'event' => $xentry->{"event"},
'subject' => $xentry->{"subject"},
'year' => $year,
'mon' => $mon,
'day' => $mday,
'hour' => $hour,
'min' => $min,
'props' => {
'current_moodid' => $moodid,
'current_mood' => $xentry->{"current_mood"},
'current_music' => $xentry->{"current_music"},
'opt_preformatted' => 0,
'opt_nocomments' => 0,
'taglist' => "",
'picture_keyword' => "",
'opt_noemail' => 0,
'unknown8bit' => 0,
'opt_backdated' => 1,
},
);
if ($xentry->{"security"} eq "public" || $xentry->{"security"} eq "private") {
$req{'security'} = $xentry->{"security"};
$req{'allowmask'} = 0;
}
elsif ($xentry->{"security"} eq "usemask" && $xentry->{"allowmask"} == 1) {
$req{'security'} = 'usemask';
$req{'allowmask'} = 1;
}
else {
$req{'security'} = 'usemask';
my @groups = ();
foreach my $grp_id (keys %rfg) {
if ($xentry->{"allowmask"}+0 & 1 << $grp_id) {
push @groups, $rfg{$grp_id}->{name};
}
}
my $mask = 0;
while (my $grpname = shift @groups) {
my $group = LJ::get_friend_group($local_u, {'name' => $grpname});
if ($group) {
$mask = $mask | (1 << $group->{groupnum});
}
}
$req{'allowmask'} = $mask;
}
my %res = ();
LJ::do_request(\%req, \%res, $flags);
if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Missing required argument") {
$warn->($res{"errmsg"} . " while processing " . $xentry->{"eventtime"});
return;
}
if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Post too large") {
$dumptofile->($req{'event'}, "large_" . $local_u->{'user'}, ".raw");
}
return $err->($xentry->{"eventtime"} . ": " . $res{"errmsg"}) unless $res{"success"} eq "OK";
$r = LJR::Distributed::store_remote_itemid(
$local_u,
$res{"itemid"},
$ru->{ru_id},
$ritem_id,
$xentry->{"itemid"});
return $err->($xentry->{"eventtime"} . ": " . $r->{"errtext"}) if $r->{"err"};
return {"err" => 0};
}
# do the actual import
sub import_journal {
my (
$throttle_speed,
$remote_site, $remote_protocol, $remote_user, $remote_pass, $remote_shared_journal,
$local_user, $local_shared_journal, $overwrite
) = @_;
$do_overwrite = $overwrite;
LJ::disconnect_dbs(); # force reconnection to the database
if ($remote_shared_journal eq "") {
$remote_shared_journal = undef;
}
if ($local_shared_journal eq "") {
$local_shared_journal = undef;
}
my %gdc_hr = ();
my %req = ();
my %lfg = ();
my %res = ();
if ($remote_protocol ne "flat" && $remote_protocol ne "xml") {
return $err->("Unsupported remote protocol $remote_protocol.");
}
$LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
$LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
$LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
$LJ::Simple::UserAgent = $LJR::USER_AGENT;
# login to the remote site
my $remote_lj = new LJ::Simple ({
site => $remote_site,
user => $remote_user,
pass => $remote_pass,
pics => 0,
moods => 0,
});
if (! defined $remote_lj) {
return $err->("Can't login to remote site.", $LJ::Simple::error);
}
if (!$remote_lj->GenerateCookie()) {
if (!$remote_lj->GenerateCookie()) {
return $err->("Can't generate login cookie.", $LJ::Simple::error);
}
}
# since we're able to login with supplied credentials --
# get and/or cache remote server and remote user ident
$ru = LJR::Distributed::get_remote_server($remote_site);
return $err->($ru->{"errtext"}) if $ru->{"err"};
# try to get userid
my $idres;
my $i1 = 0;
while(1) {
my $ua = LWPx::ParanoidAgent->new(timeout => 60);
$ua->agent($LJR::USER_AGENT);
# TODO: parameterize allpics.bml
my $url = $ru->{"servername"} . "/users/" . $remote_user . "/info/" ;
$idres = $ua->get($url);
if (!($idres && ($idres->is_success || $idres->code == 403)) && $i1 < $LJR::NETWORK_RETRIES) {
my $txt;
#foreach my $k (keys %$idres) {
# $txt .= $k . "->(" . $idres->{$k} ."), ";
#}
###_content->(500 DNS lookup timeout), _rc->(500), _headers->(HTTP::Headers=HASH(0x2d2ec70)), _msg->(DNS lookup timeout), _request->(HTTP::Request=HASH(0x2c61ac0)),
$txt .= "_msg->" . $idres->{'_msg'} . ", ";
foreach my $k (keys %{$idres->{'_headers'}}) {
$txt .= "\n" . $k . ": " . $idres->{'_headers'}->{$k} ;
}
print STDERR "*** $url $txt\n";
LJR::NETWORK_SLEEP(); $i1++; next;
}
else { last; }
}
if (!($idres && ($idres->is_success || $idres->code == 403))) {
return $err->("LWPx: Can't get remote user id: $remote_user\n");
}
if ($idres->content && $idres->content =~ /\<b\>$remote_user\<\/b\>\<\/a\>\ \((\d+)\)/s) {
$ru->{"userid"} = $1;
}
$ru->{"username"} = $remote_user;
$ru = LJR::Distributed::get_cached_user($ru); # populates $ru->{ru_id}
return $err->($ru->{"errtext"}) if $ru->{"err"};
# get local user object for user being imported into
$local_u = LJ::load_user($local_user, 1);
return $err->("Can't load local user $local_user.") unless $local_u;
$ru = LJR::Distributed::remote_local_assoc($ru, $local_u);
return $err->("error while getting remote-local association: " . $ru->{errtext})
if $ru->{err};
jstatus_print ("getting userpics");
my $e = import_pics(
$ru->{servername},
$ru->{username},
$remote_pass,
$local_user,
"", 0);
return $err->("Can't import " . $ru->{username} . ": " . $e->{errtext})
if $e->{err};
# clear duplicate protection
LJ::set_userprop($local_u, "dupsig_post", undef);
# needed everywhere
$flags = {
'u' => $local_u,
'noauth' => 1,
'BMAX_EVENT' => 150000,
'CMAX_EVENT' => 150000,
'no-cache' => 1,
'omit_underscore_check' => 1,
};
%req = ( 'mode' => 'login',
'ver' => $LJ::PROTOCOL_VER,
'clientversion' => $LJR::LJ_CLIENT,
'user' => $local_u->{'user'},
'getmenus' => 0,
);
%res = ();
LJ::do_request(\%req, \%res, $flags);
return $err->($res{'errmsg'}) unless $res{'success'} eq 'OK';
jstatus_print ("getting friend groups");
# get remote and local friend groups, mix them up, update on local server
if (! defined $remote_lj->GetFriendGroups(\%rfg)) {
return $err->("Failed to get groups on the remote site.", $LJ::Simple::error);
}
LJ::Protocol::do_request(
{
'mode' => 'getfriendgroups',
'user' => $local_u->{'user'},
'ver' => $LJ::PROTOCOL_VER,
'clientversion' => $LJR::LJ_CLIENT,
'includegroups' => 1,
'getmenus' => 0,
},
\%res,
$flags
);
if (! $res{'success'} eq "OK") {
return $err->("Unable to get local user" . $local_u->{'user'} . "groups",
$res{'success'} . ":" . $res{'errmsg'});
}
# convert it to LJ::Simple hash
while((my $k, my $v) = each %res) {
$k=~/^frgrp_([0-9]+)_(.*)$/o || next;
my ($id, $name) = ($1, $2);
if (!exists $lfg{$id}) {
$lfg{$id}={
id => $id,
public => 0,
};
}
($name eq "sortorder") && ($name="sort");
$lfg{$id}->{$name}=$v;
}
# add nonexisting remote groups (identified by name) to local server
foreach my $grp_id (keys %rfg) {
my $e = 0;
foreach my $lg (values %lfg) {
if ($lg->{name} eq $rfg{$grp_id}->{name}) {
$e = 1;
}
if ($lg->{name} =~ /default view/i) {
$e = 1;
}
}
if (!$e) {
my $egroup = 1;
foreach my $cgroup (sort { $a <=> $b } keys %lfg) {
if ($egroup == $cgroup) {
$egroup++;
}
}
if ($egroup < 31) {
$lfg{$egroup} = $rfg{$grp_id};
}
}
}
# create local friend groups (existing + copied)
my $i = 0;
%req = (
'mode' => "editfriendgroups",
'user' => $local_u->{'user'},
'clientversion' => $LJR::LJ_CLIENT,
'ver' => $LJ::PROTOCOL_VER,
);
# convert LJ::Simple hash back to ljprotocol hash
foreach my $grpid (keys %lfg) {
if ($grpid > 0 && $grpid < 31) {
my $pname = "efg_set_" . $grpid . "_name";
$req{$pname} = $lfg{$grpid}->{name};
$i++;
}
}
# do the actual request
LJ::do_request(\%req, \%res, $flags);
if (! $res{'success'} eq "OK") {
return $err->(
"Unable to update local user" . $local_u->{'user'} . "groups",
$res{'success'} . ":" . $res{'errmsg'}
);
}
# get remote days with entries
if (! defined $remote_lj->GetDayCounts(\%gdc_hr, undef)) {
return $err->("can't get day counts: ", $LJ::Simple::error);
}
# import entries by means of export.bml (XML format)
if ($remote_protocol eq "xml") {
my $mydc = {};
foreach (sort {$a<=>$b} keys %gdc_hr) {
my $timestamp = $_;
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
localtime($timestamp);
$mon++;
$year = $year + 1900;
$mydc->{$year}->{$mon} = 0;
$mydc->{$year}->{$mon} = $mydc->{$year}->{$mon} + $gdc_hr{$timestamp};
}
foreach (sort {$a <=> $b} keys %{$mydc}) {
my $y = $_;
foreach (sort {$a <=> $b} keys %{$mydc->{$y}}) {
jstatus_print ("getting XML data and creating local entries for " . $_ . "/" . $y);
my $do_login = 0;
while (1) {
if ($do_login) {
$remote_lj = new LJ::Simple ({
site => $remote_site,
user => $remote_user,
pass => $remote_pass,
pics => 0,
moods => 0,
});
if (! defined $remote_lj) {
return $err->("Can't login to remote site.", $LJ::Simple::error);
}
if (!$remote_lj->GenerateCookie()) {
if (!$remote_lj->GenerateCookie()) {
return $err->("Can't generate login cookie.", $LJ::Simple::error);
}
}
}
my $res = $remote_lj->GetRawData({
"url" => "/export_do.bml",
"post-data" => {
"authas" => $remote_user, # "nit",
"format" => "xml",
"encid" => 2, # utf-8; for full listing see htdocs/export.bml
"header" => 1,
"year" => $y,
"month" => $_,
"field_itemid" => 1,
"field_eventtime" => 1,
"field_logtime" => 1,
"field_subject" => 1,
"field_event" => 1,
"field_security" => 1,
"field_allowmask" => 1,
"field_currents" => 1,
}});
if ($res && $res->{content}) {
my $xdata = $res->{content};
LJR::unicode::force_utf8(\$xdata);
my $p1 = new XML::Parser (
Handlers => {
Start => \&xmlh_entry_start,
End => \&xmlh_entry_end,
Char => \&xmlh_entry_char
});
eval { $p1->parse($xdata); };
if ($@) {
if ($i < $LJR::NETWORK_RETRIES) {
if ($@ =~ /not\ well\-formed\ \(invalid\ token\)/) {
# $xdata <?xml version="1.0" encoding='windows-1251'?>
}
if ($xdata =~ /Login Required/) {
$do_login = 1;
}
LJR::NETWORK_SLEEP(); $i++; next;
}
else {
$dumptofile->($xdata, "err_" . $remote_user, ".xml");
return $err->("Runtime error while parsing XML data: ", $@);
}
}
if ($xmlerrt) {
$dumptofile->($xdata, "err_" . $remote_user, ".xml");
return $err->("Error while parsing XML data: ", $xmlerrt);
}
last;
}
else {
return $err->("Can't get XML data..");
}
}
}
}
}
# import entries by means of flat protocol
if ($remote_protocol eq "flat") {
# process them, day by day, sleeping a little
foreach (sort {$a<=>$b} keys %gdc_hr) {
my $timestamp = $_;
# download all the entries for a day
if ($gdc_hr{$timestamp} < $REMOTE_MAX_GET) {
jstatus_print (
"getting remote and creating local entries for " .
strftime ("%a %b %e %Y", localtime($timestamp))
);
my %r_entries=(); # remote entries
if (! defined $remote_lj->GetEntries(\%r_entries,$remote_shared_journal,"day",($timestamp))) {
if ($LJ::Simple::error =~ "Cannot display this post") {
$warn->(strftime ("%a %b %e %Y", localtime($timestamp)) . ":" . $LJ::Simple::error);
next;
}
return $err->("can't get remote entries: " . strftime ("%a %b %e %Y", localtime($timestamp)) . ": ",
$LJ::Simple::error);
}
my $rkey=undef;
my $rentry=undef;
my $r;
ENTRIES: while (($rkey, $rentry) = each(%r_entries)) {
($ritem_id, $ranum, $rhtml_id) = $remote_lj->GetItemId($rentry);
my $tevent = $remote_lj->GetEntry($rentry);
my $is_gated = LJR::Distributed::get_local_itemid($local_u, $ru->{'ru_id'}, $ritem_id, "E");
return $err->($is_gated->{"errtext"}) if $is_gated->{"err"};
next ENTRIES if $is_gated->{'itemid'};
$r = check_overwrite($local_u, $ru->{'ru_id'}, $ritem_id, $do_overwrite);
return $err->($r->{"errtext"}) if $r->{"err"};
next ENTRIES unless $r->{"continue"};
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
localtime($remote_lj->GetDate($rentry));
$mon++;
$year = $year + 1900;
LJR::Links::make_ljr_hrefs(
LJR::Links::get_server_url($ru->{"servername"}, "base"),
$ru->{"servername"},
\$tevent
);
my $tsubject = $remote_lj->GetSubject($rentry);
my $tcurrent_mood = $remote_lj->Getprop_current_mood($rentry);
my $tcurrent_music = $remote_lj->Getprop_current_music($rentry);
my $ttaglist = $remote_lj->Getprop_taglist($rentry);
$ttaglist = LJ::trim($ttaglist);
my $tpicture_keyword = $remote_lj->Getprop_picture_keyword($rentry);
# LJR::unicode::utf8ize(\$tevent);
# LJR::unicode::utf8ize(\$tsubject);
# LJR::unicode::utf8ize(\$tcurrent_mood);
# LJR::unicode::utf8ize(\$tcurrent_music);
# LJR::unicode::utf8ize(\$ttaglist);
# LJR::unicode::utf8ize(\$tpicture_keyword);
%req = ( 'mode' => 'postevent',
'ljr-import' => 1,
'ver' => $LJ::PROTOCOL_VER,
'clientversion' => $LJR::LJ_CLIENT,
'user' => $local_u->{'user'},
'username' => $local_u->{'user'},
'usejournal' => $local_u->{'user'},
'getmenus' => 0,
'lineendings' => "unix",
'event' => $tevent,
'subject' => $tsubject,
'year' => $year,
'mon' => $mon,
'day' => $mday,
'hour' => $hour,
'min' => $min,
'props' => {
'current_moodid' => $rentry->{prop_current_moodid},
'current_mood' => $tcurrent_mood,
'current_music' => $tcurrent_music,
'opt_preformatted' => $remote_lj->Getprop_preformatted($rentry),
'opt_nocomments' => $remote_lj->Getprop_nocomments($rentry),
'taglist' => $ttaglist,
'picture_keyword' => $tpicture_keyword,
'opt_noemail' => $remote_lj->Getprop_noemail($rentry),
'unknown8bit' => $remote_lj->Getprop_unknown8bit($rentry),
'opt_backdated' => 1,
},
);
my @r_protection = $remote_lj->GetProtect($rentry);
if ($r_protection[0] eq "public" || $r_protection[0] eq "private") {
$req{'security'} = $r_protection[0];
$req{'allowmask'} = 0;
}
elsif ($r_protection[0] eq "friends") {
$req{'security'} = 'usemask';
$req{'allowmask'} = 1;
}
elsif ($r_protection[0] eq "groups") {
$req{'security'} = 'usemask';
shift @r_protection;
my $mask=0;
while (my $grpname = shift @r_protection) {
my $group = LJ::get_friend_group($local_u, {'name' => $grpname});
$mask = $mask | (1 << $group->{groupnum});
}
$req{'allowmask'} = $mask;
}
%res = ();
LJ::do_request(\%req, \%res, $flags);
if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Post too large") {
$dumptofile->($req{'event'}, "large_" . $local_u->{'user'}, ".raw");
}
if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Invalid text encoding") {
$warn->($res{"errmsg"});
next;
}
if ($res{"success"} ne "OK" && $res{"errmsg"} =~ "Invalid or malformed tag list") {
return $err->($res{"errmsg"} . ": [$ttaglist]");
}
return $err->($res{'errmsg'}) unless $res{'success'} eq 'OK';
$r = LJR::Distributed::store_remote_itemid(
$local_u,
$res{"itemid"},
$ru->{ru_id},
$ritem_id,
$rhtml_id);
return $err->($r->{"errtext"}) if $r->{"err"};
}
sleep($throttle_speed);
}
else {
$warn->("Too much entries for a day. " . $local_u->{'user'} . " " .
strftime ("%a %b %e %Y", localtime($timestamp))
);
}
} # process them day by day
}
if ($warns) {
my %warns = ('warns' => $warns);
return \%warns;
}
else {
return undef;
}
}
return 1;

533
local/bin/ljrimport/ipics.pl Executable file
View File

@@ -0,0 +1,533 @@
#!/usr/bin/perl
use strict;
use Image::Size ();
use Simple; # corrected LJ::Simple
require "$ENV{'LJHOME'}/cgi-bin/ljlib.pl";
require "ljr-defaults.pl";
require "ljr-links.pl";
require LJR::Distributed;
require LWPx::ParanoidAgent;
# error handling
my $err = sub {
my %res = ();
my $cstack = "\ncallstack:";
my $i = 0;
while ( 1 ) {
my $tfunc = (caller($i))[3];
if ($tfunc && $tfunc ne "") {
if ($tfunc !~ /\_\_ANON\_\_/) {
$cstack .= " " . $tfunc;
}
$i = $i + 1;
}
else {
last;
}
}
$res{"err"} = 1;
$res{"errtext"} = join ("\n", @_);
$res{"errtext"} .= $cstack;
return \%res;
};
# example:
my $DEBUG = 0;
sub cache_remote_pics {
my ($remote_site, $remote_user, $remote_pass, $local_userid) = @_;
my $ua;
my $res;
my %remote_urls = ();
my %remote_keywords = ();
my %remote_comments = ();
my $default_pic = "";
my $i = 0;
my $content;
# get remote pictures list with keywords
if ($remote_pass ne "") {
$LJ::Simple::network_retries = $LJR::NETWORK_RETRIES;
$LJ::Simple::network_sleep = $LJR::NETWORK_SLEEP;
$LJ::Simple::LJ_Client = $LJR::LJ_CLIENT;
$LJ::Simple::UserAgent = $LJR::USER_AGENT;
my $ljs_site = $remote_site;
if ($ljs_site =~ /^http\:\/\/(.*)/) {
$ljs_site = $1;
}
my $remote_lj = new LJ::Simple ({
site => $ljs_site,
user => $remote_user,
pass => $remote_pass,
pics => 0,
moods => 0,
});
return $err->("Can't login to remote site.", $LJ::Simple::error)
unless defined($remote_lj);
if (!$remote_lj->GenerateCookie()) {
return $err->("Can't generate login cookie.", $LJ::Simple::error);
}
$res = $remote_lj->GetRawData({
"url" => "/allpics.bml",
});
if (!($res && $res->{content})) {
return $err->("LJ::Simple: Can't get remote user pictures: $remote_user\n");
}
$content = $res->{content};
}
else {
while(1) {
$ua = LWPx::ParanoidAgent->new(timeout => 60);
$ua->agent($LJR::USER_AGENT);
# TODO: parameterize allpics.bml
$res = $ua->get($remote_site . "/allpics.bml?user=" . $remote_user);
if (!($res && $res->is_success) && $i < $LJR::NETWORK_RETRIES) {
LJR::NETWORK_SLEEP(); $i++; next;
}
else {
last;
}
}
if (!($res && $res->is_success)) {
return $err->("LWPx: Can't get remote user pictures: $remote_user\n");
}
$content = $res->content;
}
my $ru = LJR::Distributed::get_remote_server($remote_site);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$ru->{username} = $remote_user;
$ru = LJR::Distributed::get_cached_user($ru);
$i = 0;
my $dbh = LJ::get_db_writer();
return $err->("Can't get database writer!") unless $dbh;
$dbh->do("DELETE FROM ljr_cached_userpics WHERE ru_id=?", undef, $ru->{ru_id});
return $err->($dbh->errstr) if $dbh->err;
my $iru;
my $userpic_base = LJR::Links::get_server_url($remote_site, "userpic_base");
# extract pic urls and keywords
if ($content =~ m!<\s*?body.*?>(.+)</body>!si) {
$content = $1;
while ($content =~
/\G.*?($userpic_base\/(\d+)\/(\d+))(.*?)($userpic_base\/(\d+)\/(\d+)|$)(.*)/sg
) {
my $picurl = $1;
my $props = $4;
my $cuserid = $3;
my $cpicid = $2;
$content = $5 . $8;
my $is_default = 0;
# save userid
if (!$iru->{ru_id}) {
$iru = LJR::Distributed::get_remote_server($remote_site);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$iru->{username} = $remote_user;
$iru->{userid} = $cuserid;
$iru = LJR::Distributed::get_cached_user($iru);
return $err->($ru->{"errtext"}) if $ru->{"err"};
}
if ($props =~ /(.*?)Keywords\:\<\/b\>\ (.*?)\<br\ \/\>(.*?)\<\/td\>/s) {
$remote_keywords{$picurl} = $2;
$remote_comments{$picurl} = $3;
$remote_comments{$picurl} =~ s/^\s+|\s+$//;
}
if ($props =~ /\<u\>Default\<\/u\>/s) {
$default_pic = $picurl;
$is_default = 1;
}
my @keywords = "";
if ($remote_keywords{$picurl}) {
@keywords = split(/\s*,\s*/, $remote_keywords{$picurl});
@keywords = grep { s/^\s+//; s/\s+$//; $_; } @keywords;
}
elsif ($is_default) {
@keywords = ("");
}
foreach my $kw (@keywords) {
if($remote_urls{$cpicid}) {
$dbh->do("UPDATE ljr_cached_userpics set keyword=?, is_default=?, comments=?
where ru_id=? and remote_picid=?",
undef, $kw, $is_default, $remote_comments{$picurl},
$ru->{ru_id}, $cpicid);
return $err->($dbh->errstr) if $dbh->err;
}
else {
$dbh->do("INSERT INTO ljr_cached_userpics VALUES (?,?,?,?,?)",
undef, $ru->{ru_id}, $cpicid, $kw,
$is_default, $remote_comments{$picurl});
return $err->($dbh->errstr) if $dbh->err;
}
}
$remote_urls{$cpicid} = $picurl;
}
}
return undef;
}
sub import_pics {
my (
$remote_site, $remote_user, $remote_pass,
$local_user, $o_keyword, $o_default
) = @_;
my $MAX_UPLOAD = 40960;
my %remote_ids = ();
my %remote_urls = ();
my %remote_keywords = ();
my %remote_comments = ();
my $default_pic = "";
my $ru = LJR::Distributed::get_remote_server($remote_site);
return $err->($ru->{"errtext"}) if $ru->{"err"};
$ru->{username} = $remote_user;
$ru = LJR::Distributed::get_cached_user($ru);
# load user object (force, otherwise get error outside of apache)
my $u = LJ::load_user($local_user, 1);
return $err->("Invalid local user: " . $local_user) unless $u;
# prepare database connections (for different versions of user objects)
my ($dbcm, $dbcr, $sth);
$dbcm = LJ::get_cluster_master($u);
return $err->("Can't get cluster master!") unless $dbcm;
$dbcr = LJ::get_cluster_def_reader($u);
return $err->("Can't get cluster reader!") unless $dbcr;
my $dbh = LJ::get_db_writer();
return $err->("Can't get database writer!") unless $dbh;
my $dbr = LJ::get_db_reader();
return $err->("Can't get database reader!") unless $dbr;
my $e;
if (!$o_keyword && !$o_default) {
$e = cache_remote_pics($remote_site, $remote_user, $remote_pass, $u->{userid});
return $e if $e->{err};
}
else {
$sth = $dbr->prepare(
"SELECT ru_id FROM ljr_cached_userpics WHERE ru_id=? GROUP BY ru_id");
$sth->execute($ru->{ru_id});
my $ruid = $sth->fetchrow_hashref;
$sth->finish;
if (!$ruid) {
$e = cache_remote_pics($remote_site, $remote_user, $remote_pass, $u->{userid});
return $e if $e->{err};
}
}
# get ru->{userid} which should come up after caching remote pic props
$ru = LJR::Distributed::get_cached_user($ru);
if ($o_keyword) {
$sth = $dbr->prepare(
"SELECT remote_picid, keyword, is_default, comments " .
"FROM ljr_cached_userpics WHERE ru_id=? and keyword=?");
$sth->execute($ru->{ru_id}, $o_keyword);
}
elsif ($o_default) {
$sth = $dbr->prepare(
"SELECT remote_picid, keyword, is_default, comments " .
"FROM ljr_cached_userpics WHERE ru_id=? and is_default=1");
$sth->execute($ru->{ru_id});
}
else {
$sth = $dbr->prepare(
"SELECT remote_picid, keyword, is_default, comments " .
"FROM ljr_cached_userpics WHERE ru_id=?");
$sth->execute($ru->{ru_id});
}
my $i = 0;
while (my $rpic = $sth->fetchrow_hashref) {
my $picurl = $remote_site . "/userpic/" . $rpic->{remote_picid} . "/" . $ru->{userid};
$remote_ids{$i} = $rpic->{remote_picid};
$remote_urls{$rpic->{remote_picid}} = $picurl;
$remote_comments{$picurl} = $rpic->{comments};
$remote_keywords{$picurl} =
(($remote_keywords{$picurl}) ? $remote_keywords{$picurl} . "," : "") .
$rpic->{keyword};
if ($rpic->{is_default}) {
$default_pic = $picurl;
}
print
$picurl . ":" .
$remote_ids{$i} . ":" .
$remote_comments{$picurl} . ":" .
$remote_keywords{$picurl} . "\n"
if $DEBUG;
$i++;
}
$sth->finish;
RPICID: foreach my $rpicid (sort {$a <=> $b} values %remote_ids) {
my $local_picid = $dbr->selectrow_array(
"SELECT local_picid FROM ljr_remote_userpics " .
"WHERE ru_id=? and remote_picid=? and local_userid = ?",
undef, $ru->{ru_id}, $rpicid, $u->{userid});
if ($local_picid) {
my $r_picid = $dbr->selectrow_array(
"SELECT picid FROM userpic2 WHERE picid=?",
undef, $local_picid);
if (!$r_picid) {
$u->do("DELETE FROM ljr_remote_userpics WHERE local_picid=?", undef, $local_picid);
$local_picid = undef;
}
else {
next RPICID;
}
}
my %POST = ();
$POST{urlpic} = $remote_urls{$rpicid};
$POST{keywords} = $remote_keywords{$remote_urls{$rpicid}};
$POST{comments} = $remote_comments{$remote_urls{$rpicid}};
$POST{url} = "";
if ($default_pic eq $remote_urls{$rpicid}) {
$POST{make_default} = 1;
}
# get remote picture and validate it
my $ua;
my $res;
my ($sx, $sy, $filetype);
$i = 0;
while(1) {
$ua = LWPx::ParanoidAgent->new(
timeout => 60,
max_size => $MAX_UPLOAD + 1024);
$ua->agent($LJR::USER_AGENT);
$res = $ua->get($POST{urlpic});
# if the picture doesn't exist on the remote server
# then we get 404 http error and remove it from our cache
if ($res &&
($res->{"_rc"} eq 404 || $res->{"_rc"} eq 503)
) {
$dbh->do("DELETE FROM ljr_cached_userpics WHERE ru_id=? and remote_picid=?",
undef, $ru->{ru_id}, $rpicid);
return $err->($dbh->errstr) if $dbh->err;
next RPICID;
}
$POST{userpic} = $res->content if $res && $res->is_success;
($sx, $sy, $filetype) = Image::Size::imgsize(\$POST{'userpic'});
if (!(
$res && $res->is_success && defined($sx) &&
length($POST{'userpic'}) <= $MAX_UPLOAD &&
($filetype eq "GIF" || $filetype eq "JPG" || $filetype eq "PNG") &&
$sx <= 100 && $sy <= 100
) &&
$i < $LJR::NETWORK_RETRIES) {
LJR::NETWORK_SLEEP(); $i++; next;
}
else {
last;
}
}
if (!($res && $res->is_success)) {
return $err->("Can't get remote user picture: ",
$remote_user, $local_user, $o_keyword, $o_default, $POST{urlpic},
$res->status_line);
}
if (!defined $sx) {
print ("Invalid image: " . $POST{urlpic} . "\n");
next RPICID;
}
if (length($POST{'userpic'}) > $MAX_UPLOAD) {
return $err->("Picture " . $POST{urlpic} . "is too large");
}
return $err->("Unsupported filetype: " . $POST{urlpic})
unless ($filetype eq "GIF" || $filetype eq "JPG" || $filetype eq "PNG");
return $err->("Image too large: " . $POST{urlpic}) if ($sx > 150 || $sy > 150);
my $base64 = Digest::MD5::md5_base64($POST{'userpic'});
# see if it's a duplicate
my $picid;
my $contenttype;
if ($filetype eq "GIF") { $contenttype = 'G'; }
elsif ($filetype eq "PNG") { $contenttype = 'P'; }
elsif ($filetype eq "JPG") { $contenttype = 'J'; }
$picid = $dbcr->selectrow_array(
"SELECT picid FROM userpic2 WHERE userid=? AND fmt=? AND md5base64=?",
undef, $u->{'userid'}, $contenttype, $base64);
$picid = 0 unless defined($picid);
print "trying to insert into db\n" if $DEBUG;
# if picture isn't a duplicate, insert it
if ($picid == 0) {
# Make a new global picid
$picid = LJ::alloc_global_counter('P') or
return $err->('Unable to allocate new picture id');
$u->do(
"INSERT INTO userpic2 (picid, userid, fmt, width, height, " .
"picdate, md5base64, location, state) " .
"VALUES (?, ?, ?, ?, ?, NOW(), ?, ?, 'N')",
undef, $picid, $u->{'userid'}, $contenttype, $sx, $sy, $base64, undef);
return $err->($u->errstr) if $u->err;
my $clean_err = sub {
if ($picid) {
$u->do(
"DELETE FROM userpic2 WHERE userid=? AND picid=?",
undef, $u->{'userid'}, $picid);
$u->do(
"DELETE FROM userpicblob2 WHERE userid=? AND picid=?",
undef, $u->{'userid'}, $picid);
}
return $err->(@_);
};
### insert the blob
$u->do(
"INSERT INTO userpicblob2 (userid, picid, imagedata) VALUES (?,?,?)",
undef, $u->{'userid'}, $picid, $POST{'userpic'});
return $clean_err->($u->errstr) if $u->err;
# make it their default pic?
if ($POST{'make_default'}) {
LJ::update_user($u, { defaultpicid => $picid });
$u->{'defaultpicid'} = $picid;
}
# set default keywords?
if ($POST{'keywords'} && $POST{'keywords'} ne '') {
print "storing keywords\n" if $DEBUG;
$sth = $dbcr->prepare("SELECT kwid, picid FROM userpicmap2 WHERE userid=?");
$sth->execute($u->{'userid'});
my @exist_kwids;
while (my ($kwid, $picid) = $sth->fetchrow_array) {
$exist_kwids[$kwid] = $picid;
}
my @keywords = split(/\s*,\s*/, $POST{'keywords'});
@keywords = grep { s/^\s+//; s/\s+$//; $_; } @keywords;
my (@bind, @data);
my $c = 0;
foreach my $kw (@keywords) {
my $kwid = LJ::get_keyword_id($u, $kw);
next unless $kwid; # Houston we have a problem! This should always return an id.
if ($c > $LJ::MAX_USERPIC_KEYWORDS) {
return $clean_err->("Too many userpic keywords: " . LJ::ehtml($kw));
}
if ($exist_kwids[$kwid]) { # Already used on another picture
# delete existing pic while there's newer one
$u->do("
delete ljr_remote_userpics, ljr_cached_userpics
from ljr_cached_userpics, ljr_remote_userpics
where
ljr_cached_userpics.ru_id = ljr_remote_userpics.ru_id and
ljr_cached_userpics.remote_picid = ljr_remote_userpics.remote_picid and
ljr_remote_userpics.local_userid = ? and local_picid = ?",
undef, $u->{'userid'}, $exist_kwids[$kwid]);
$u->do("DELETE FROM userpicmap2 WHERE userid=? AND picid=?",
undef, $u->{'userid'}, $exist_kwids[$kwid]);
$u->do("DELETE FROM userpicblob2 WHERE userid=? AND picid=?",
undef, $u->{'userid'}, $exist_kwids[$kwid]);
$u->do("DELETE FROM userpic2 WHERE userid=? AND picid=?",
undef, $u->{'userid'}, $exist_kwids[$kwid]);
}
push @bind, '(?, ?, ?)';
push @data, $u->{'userid'}, $kwid, $picid;
$c++;
}
if (@data && @bind) {
my $bind = join(',', @bind);
$u->do(
"INSERT INTO userpicmap2 (userid, kwid, picid) VALUES $bind",
undef, @data);
}
}
# set default comments and the url
my (@data, @set);
if ($POST{'comments'} && $POST{'comments'} ne '') {
push @set, 'comment=?';
push @data, LJ::text_trim($POST{'comments'}, $LJ::BMAX_UPIC_COMMENT, $LJ::CMAX_UPIC_COMMENT);
}
if ($POST{'url'} ne '') {
push @set, 'url=?';
push @data, $POST{'url'};
}
if (@set) {
my $set = join(',', @set);
$u->do("UPDATE userpic2 SET $set WHERE userid=? AND picid=?",
undef, @data, $u->{'userid'}, $picid);
return $err->($u->errstr) if $u->err;
}
$u->do("INSERT INTO ljr_remote_userpics VALUES (?,?,?,?)",
undef, $ru->{ru_id}, $rpicid, $u->{userid}, $picid);
return $err->($u->errstr) if $u->err;
}
}
return undef;
}
return 1;

35
local/bin/ljrimport/ljr-defaults.pl Executable file
View File

@@ -0,0 +1,35 @@
#!/usr/bin/perl
package LJR;
$LJ_CLIENT = "LJR::Import/0.01";
$USER_AGENT = "LJR::Import/0.01; http://lj.rossia.org/; lj-admin\@rossia.org";
# How much times to retry if any network related error occurs
$NETWORK_RETRIES = 10; #was: 20
# Hom much seconds to wait before each retry
$NETWORK_SLEEP = 30; #was: 5
$DEBUG = 1;
sub NETWORK_SLEEP {
my $msg = shift;
if ($msg) {
$msg = " (" . $msg . ")";
}
else {
$msg = "";
}
my $t = `date +"%D %T"`;
print
substr($t, 0, length($t) - 1) .
" sleeping $NETWORK_SLEEP seconds due to network related error" . $msg . ".\n"
if $DEBUG;
sleep $NETWORK_SLEEP;
};
return 1;

106
local/bin/ljrimport/ljr-import.pl Executable file
View File

@@ -0,0 +1,106 @@
#!/usr/bin/perl -w
package LJR::Import;
use strict; # preventing my program from doing bad things
use DBI; # http://dbi.perl.org
use POSIX ();
do $ENV{'LJHOME'} . "/cgi-bin/ljconfig.pl";
my $qhost = $LJ::DBINFO{'master'}->{'host'};
my $quser = $LJ::DBINFO{'master'}->{'user'};
my $qpass = $LJ::DBINFO{'master'}->{'pass'};
my $qdb = $LJ::DBINFO{'master'}->{'dbname'};
my $qsock = $LJ::DBINFO{'master'}->{'sock'};
my $qport = $LJ::DBINFO{'master'}->{'port'};
$| = 1; # unbuffered (almost) output
# global database handle
$LJR::Import::global_dbh = 0;
# global shutdown request received flag
$LJR::Import::cool_stop = 0;
my $history_id;
# POSIX unmasks the sigprocmask properly
my $sigset = POSIX::SigSet->new();
my $action = POSIX::SigAction->new(
'LJR::Import::sigTERM_handler', $sigset, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGTERM, $action);
sub log_print {
my $msg = shift;
my $t = `date +"%D %T"`;
print substr($t, 0, length($t) - 1) . " $msg\n";
}
sub sigTERM_handler {
my $t = `date +"%D %T"`;
print substr($t, 0, length($t) - 1) . " ljr-import.pl: received shutdown request\n";
$LJR::Import::cool_stop = 1;
}
# configuration
my $speed_throttle = 10; # seconds
print "\n";
LJR::Import::log_print("started");
# main loop, throttled
while (!process_exit()) {
process_queue(); # process new requests for import if any
sleep ($speed_throttle); # sleep for a while
}
LJR::Import::log_print("ljr-import.pl: shutting down due to safe shutdown request");
sub import_log {
my ($istatus) = @_;
my $sth2 = $LJR::Import::global_dbh->prepare (
"update ljr_ihistory set
istatus = ?,
idate = now()
where importid = ?");
$sth2->execute($istatus, $history_id);
$sth2->finish;
}
sub process_exit {
return ($LJR::Import::cool_stop);
}
sub process_queue {
my $row;
my $row1;
my $sth2;
my $e;
$LJR::Import::global_dbh = DBI->connect(
"DBI:mysql:mysql_socket=$qsock;hostname=$qhost;port=$qport;database=$qdb",
$quser, $qpass,
{RaiseError => 0, AutoCommit => 1}
) || die "Can't open database connection: $DBI::errstr";
my $sth = $LJR::Import::global_dbh->prepare("SELECT * from ljr_iqueue order by priority, importid");
$sth->execute;
while (($row = $sth->fetchrow_hashref) && !process_exit()) {
my $r = system ("nice -n 19 ./ljr-importdo.pl");
if ($r != 0) {
$sth->finish;
$LJR::Import::global_dbh->disconnect;
return;
}
sleep($speed_throttle); # do not hurry
$sth->execute; # refresh the query
}
$sth->finish;
$LJR::Import::global_dbh->disconnect;
}

218
local/bin/ljrimport/ljr-importdo.pl Executable file
View File

@@ -0,0 +1,218 @@
#!/usr/bin/perl -w
package LJR::Import;
use strict; # preventing my program from doing bad things
use DBI; # http://dbi.perl.org
use POSIX ();
do $ENV{'LJHOME'} . "/cgi-bin/ljconfig.pl";
my $qhost = $LJ::DBINFO{'master'}->{'host'};
my $quser = $LJ::DBINFO{'master'}->{'user'};
my $qpass = $LJ::DBINFO{'master'}->{'pass'};
my $qdb = $LJ::DBINFO{'master'}->{'dbname'};
my $qsock = $LJ::DBINFO{'master'}->{'sock'};
my $qport = $LJ::DBINFO{'master'}->{'port'};
require "ijournal.pl";
require "icomments.pl";
$| = 1; # unbuffered (almost) output
# global database handle
$LJR::Import::global_dbh = 0;
# global shutdown request received flag
$LJR::Import::cool_stop = 0;
my $history_id;
# POSIX unmasks the sigprocmask properly
my $sigset = POSIX::SigSet->new();
my $action = POSIX::SigAction->new(
'LJR::Import::sigTERM_handler', $sigset, &POSIX::SA_NODEFER);
POSIX::sigaction(&POSIX::SIGTERM, $action);
sub log_print {
my $msg = shift;
my $t = `date +"%D %T"`;
print substr($t, 0, length($t) - 1) . " $msg\n";
}
sub sigTERM_handler {
my $t = `date +"%D %T"`;
print substr($t, 0, length($t) - 1) . " ljr-import-do.pl: received shutdown request\n";
$LJR::Import::cool_stop = 1;
}
# configuration
my $speed_throttle = 10; # seconds
process_queue_alone(); # there must be something there!
if (process_exit()) {
exit 1;
}
else {
exit 0;
}
sub import_log {
my ($istatus) = @_;
my $sth2 = $LJR::Import::global_dbh->prepare (
"update ljr_ihistory set
istatus = ?,
idate = now()
where importid = ?");
$sth2->execute($istatus, $history_id);
$sth2->finish;
}
sub process_exit {
return ($LJR::Import::cool_stop);
}
sub process_queue_alone {
my $row;
my $row1;
my $sth2;
my $e;
$LJR::Import::global_dbh = DBI->connect(
"DBI:mysql:mysql_socket=$qsock;hostname=$qhost;port=$qport;database=$qdb",
$quser, $qpass,
{RaiseError => 0, AutoCommit => 1}
) || die "Can't open database connection: $DBI::errstr";
my $sth = $LJR::Import::global_dbh->prepare("SELECT * from ljr_iqueue order by priority, importid");
$sth->execute;
while (($row = $sth->fetchrow_hashref) && !process_exit()) {
my $sth1 = $LJR::Import::global_dbh->prepare (
"SELECT * from ljr_iqueue where
local_user = '" . $row->{'local_user'} . "'
order by local_user desc, importid desc limit 1");
$sth1->execute; # find last user request for import
$row = $sth1->fetchrow_hashref;
# create history record
$sth2 = $LJR::Import::global_dbh->prepare (
"insert into ljr_ihistory values ('',?,?,'',?,?,?,?,now(),'STARTED',now())");
$sth2->execute (
$row->{'remote_site'}, $row->{'remote_user'}, $row->{'remote_protocol'},
$row->{'local_user'}, $row->{'opt_overwrite'}, $row->{'opt_comments'}
); # save import history (parameters, time when started)
$history_id = $LJR::Import::global_dbh->selectrow_array("SELECT LAST_INSERT_ID()");
$sth2->finish;
LJR::Import::log_print(
$row->{'local_user'} . " <- " .
$row->{'remote_site'} . "::" . $row->{'remote_user'} .
" (entries)"
);
$e = "";
$e = import_journal(
0, # throttle_speed (seconds)
$row->{'remote_site'}, # remote_site
$row->{'remote_protocol'},# remote_protocol
$row->{'remote_user'}, # remote_user
$row->{'remote_pass'}, # remote_pass
"", # remote shared journal (if any)
$row->{'local_user'}, # local_user
"", # local shared journal (if any)
$row->{'opt_overwrite'} # overwrite entries
);
if ($row->{'opt_comments'} && (!$e || !$e->{'err'})) {
LJR::Import::log_print(
$row->{'local_user'} . " <- " .
$row->{'remote_site'} . "::" . $row->{'remote_user'} .
" (caching comments)"
);
$e = get_comments(
$row->{'remote_site'},
$row->{'remote_user'},
$row->{'remote_pass'},
1);
if (!$e || !$e->{'err'}) {
LJR::Import::log_print(
$row->{'local_user'} . " <- " .
$row->{'remote_site'} . "::" . $row->{'remote_user'} .
" (creating comments)");
$e = create_imported_comments (
$row->{'remote_site'},
$row->{'remote_user'},
$row->{'local_user'});
}
}
if ($e->{'err'}) {
$sth2 = $LJR::Import::global_dbh->prepare (
"update ljr_ihistory " .
"set remote_pass = '" . $row->{'remote_pass'} . "' " .
"where importid = " . $history_id . " ;"
);
$sth2->execute; # save remote pass for debugging purposes
$sth2->finish;
my $boo = $e->{errtext};
$boo =~ s/\n//;
LJR::Import::log_print(
$row->{'local_user'} . " <- " .
$row->{'remote_site'} . "::" . $row->{'remote_user'} . " " . $boo
);
import_log($e->{errtext});
}
else {
$sth2 = $LJR::Import::global_dbh->prepare (
"update ljr_ihistory " .
"set remote_pass = '' " .
"where remote_site = '" . $row->{'remote_site'} . "' and " .
"remote_user = '" . $row->{'remote_user'} . "' ;"
);
$sth2->execute; # remove remote pass since the journal was imported successfully
$sth2->finish;
LJR::Import::log_print(
$row->{'local_user'} . " <- " .
$row->{'remote_site'} . "::" . $row->{'remote_user'} .
": successful"
);
if ($e->{'warns'}) {
import_log("SUCCESSFUL, but " . $e->{'warns'});
}
else {
import_log("SUCCESSFUL");
}
}
$sth1 = $LJR::Import::global_dbh->prepare (
"delete from ljr_iqueue " .
"where local_user = '" . $row->{'local_user'} . "'"
); # empty all the user's request after processing last one
$sth1->execute;
$sth1->finish;
$sth->finish;
# we're quitting!
# if (process_exit()) {
$LJR::Import::global_dbh->disconnect;
return;
# }
}
$sth->finish;
$LJR::Import::global_dbh->disconnect;
}

117
local/bin/ljrimport/ljr-links.pl Executable file
View File

@@ -0,0 +1,117 @@
#!/usr/bin/perl
use strict;
package LJR::Links;
sub get_server_url {
my ($canonical_url, $type) = @_;
if ($canonical_url eq "http://www.livejournal.com") {
if ($type eq "base") {
return "livejournal.com";
}
if ($type eq "userpic_base") {
return "http://userpic.livejournal.com";
}
}
}
sub make_ljr_hrefs {
my ($server_patt, $server_full, $text) = @_;
my $content = $$text;
$$text = "";
my $url;
my $orig_url;
my $orig_url_text;
return unless $content;
# replace valid html hyperlinks (<a href=http://www.livejournal.com/users/username/111.html>url_text</a>)
# with <ljr-href url="/users/username/111.html" site="http://www.livejournal.com">url_text</ljr-href>
#
while ($content =~
/\G(.*?)(\<a.*?href.*?=(\s?\"\s?)?(.*?)(\s?\"\s?)?\>(.*?)\<\/a\>)(.*)/sgi
) {
$$text .= $1;
$orig_url = $2;
$orig_url_text = $6;
$url = $4;
$content = $7;
# relative link (to the server from which we're importing)
if ($url =~ /^(\/users\/.*?\/\d*?\.html(.*?$))/) { # (\?thread=\d*\#t\d*)|$
$$text .= "<ljr-href url=\"$1\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
# relative link to oldstyle talkread.bml
elsif ($url =~ /^\/talkread.bml?\?journal=(.*?)\&itemid=(\d+)/) {
$$text .= "<ljr-href url=\"/users/$1/$2.html\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
# absolute link to oldstyle talkread.bml
elsif ($url =~ /^http:\/\/(www\.|)$server_patt\/talkread.bml\?journal=(.*?)\&itemid=(\d+)/) {
$$text .= "<ljr-href url=\"/users/$2/$3.html\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
# free users own two types of urls (first is canonical)
# http://www.livejournal.com/users/free_user/123456.html
# http://www.livejournal.com/~free_user/123456.html
elsif ($url =~ /^http:\/\/(www\.|)$server_patt(((\/~(\w*?)\/)|(\/users\/.*?\/))(\d*?\.html(.*?$)))/) { # (\?thread=\d*\#t\d*)|$
if ($5) {
$$text .= "<ljr-href url=\"/users/$5/$7\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
else {
$$text .= "<ljr-href url=\"$2\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
}
# payed users might own http://payeduser.livejournal.com/123456.html urls
elsif ($url =~ /^http:\/\/(\w*?)\.$server_patt\/(\d*?\.html(.*?$))/) { # (\?thread=\d*\#t\d*)|$
$$text .= "<ljr-href url=\"/users/$1/$2\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
else {
$$text .= $orig_url;
}
}
$$text .= $content;
$content = $$text;
$$text = "";
# replace strings like http://www.livejournal.com/users/lookslikeentry/123456.html with
# <ljr-href url="/users/lookslikeentry/123456.html" site="http://www.livejournal.com">http://www.livejournal.com/users/lookslikeentry/123456.html</ljr-href>
#
# now these can be only absolute links starting with http://
while ($content =~
/\G(.*?(^|[\ \t\r\n\f]))(http:\/\/.*?)(($|[\ \t\r\n\f]).*)/sg
) {
$$text .= $1;
$orig_url = $3;
$orig_url_text = $3;
$url = $3;
$content = $4;
# free users (copied from above)
if ($url =~ /^http:\/\/(www\.|)$server_patt(((\/~(\w*?)\/)|(\/users\/.*?\/))(\d*?\.html(.*?$)))/) { # (\?thread=\d*\#t\d*)|$
if ($5) {
$$text .= "<ljr-href url=\"/users/$5/$7\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
else {
$$text .= "<ljr-href url=\"$2\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
}
# oldstyle talkread.bml
elsif ($url =~ /^http:\/\/(www\.|)$server_patt\/talkread.bml\?journal=(.*?)\&itemid=(\d+)/) {
$$text .= "<ljr-href url=\"/users/$2/$3.html\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
# payed users (copied from above)
elsif ($url =~ /^http:\/\/(\w*?)\.$server_patt\/(\d*?\.html(.*?$))/) { # (\?thread=\d*\#t\d*)|$
$$text .= "<ljr-href url=\"/users/$1/$2\" site=\"$server_full\">$orig_url_text</ljr-href>";
}
else {
$$text .= $orig_url;
}
}
$$text .= $content;
}
return 1;

17
local/bin/ljrimport/ljr-start.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/bin/bash
export LJHOME=/home/lj-admin/lj
IMPORT_NAME=ljr-import
ipid=`ps -e --format=pid,cmd | grep $IMPORT_NAME | grep -v grep | cut --bytes=1-5`
if [ ! "$ipid" == "" ]; then
echo "LJR::Import found, PID: $ipid; shutdown with ljr-stop.sh first."
else
if [ "$LJHOME" != "" ]; then
cd $LJHOME/bin/ljrimport
./ljr-import.pl >> $LJHOME/logs/ljr-import.log 2>&1 &
else
echo \$LJHOME is not set.
fi
fi

12
local/bin/ljrimport/ljr-stop.sh Executable file
View File

@@ -0,0 +1,12 @@
#!/bin/bash
IMPORT_NAME=ljr-import
ipid=`ps -e --format=pid,cmd | grep $IMPORT_NAME | grep -v grep | cut --bytes=1-5`
if [ ! "$ipid" == "" ]; then
echo "LJR::Import found, PID: $ipid; sending shutdown signal."
kill $ipid
else
echo "LJR::Import is not running."
fi