353 lines
11 KiB
Perl
353 lines
11 KiB
Perl
|
#!/usr/bin/perl
|
||
|
|
#
|
||
|
|
|
||
|
|
use strict;
|
||
|
|
use Getopt::Long;
|
||
|
|
$| = 1;
|
||
|
|
|
||
|
|
use lib "$ENV{'LJHOME'}/cgi-bin/";
|
||
|
|
require "ljlib.pl";
|
||
|
|
use LJ::Blob;
|
||
|
|
use LJ::User;
|
||
|
|
|
||
|
|
use constant DEBUG => 0; # turn on for debugging (mostly db handle crap)
|
||
|
|
|
||
|
|
my $BLOCK_MOVE = 5000; # user rows to get at a time before moving
|
||
|
|
my $BLOCK_INSERT = 25; # rows to insert at a time when moving users
|
||
|
|
my $BLOCK_UPDATE = 1000; # users to update at a time if they had no data to move
|
||
|
|
|
||
|
|
# get options
|
||
|
|
my %opts;
|
||
|
|
exit 1 unless
|
||
|
|
GetOptions("lock=s" => \$opts{locktype},
|
||
|
|
"user=s" => \$opts{user},
|
||
|
|
"total=i" => \$opts{total},);
|
||
|
|
|
||
|
|
|
||
|
|
# if no locking, notify them about it
|
||
|
|
die "ERROR: Lock must be of types 'ddlockd' or 'none'\n"
|
||
|
|
if $opts{locktype} && $opts{locktype} !~ m/^(?:ddlockd|none)$/;
|
||
|
|
|
||
|
|
# used for keeping stats notes
|
||
|
|
my %stats = (); # { 'stat' => 'value' }
|
||
|
|
|
||
|
|
my %handle;
|
||
|
|
|
||
|
|
# database handle retrieval sub
|
||
|
|
my $get_db_handles = sub {
|
||
|
|
# figure out what cluster to load
|
||
|
|
my $cid = shift(@_) + 0;
|
||
|
|
|
||
|
|
my $dbh = $handle{0};
|
||
|
|
unless ($dbh) {
|
||
|
|
$dbh = $handle{0} = LJ::get_dbh({ raw => 1 }, "master");
|
||
|
|
print "Connecting to master ($dbh)...\n";
|
||
|
|
eval {
|
||
|
|
$dbh->do("SET wait_timeout=28800");
|
||
|
|
};
|
||
|
|
$dbh->{'RaiseError'} = 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
my $dbcm;
|
||
|
|
$dbcm = $handle{$cid} if $cid;
|
||
|
|
if ($cid && ! $dbcm) {
|
||
|
|
$dbcm = $handle{$cid} = LJ::get_cluster_master({ raw => 1 }, $cid);
|
||
|
|
print "Connecting to cluster $cid ($dbcm)...\n";
|
||
|
|
return undef unless $dbcm;
|
||
|
|
eval {
|
||
|
|
$dbcm->do("SET wait_timeout=28800");
|
||
|
|
};
|
||
|
|
$dbcm->{'RaiseError'} = 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
# return one or both, depending on what they wanted
|
||
|
|
return $cid ? ($dbh, $dbcm) : $dbh;
|
||
|
|
};
|
||
|
|
|
||
|
|
# percentage complete
|
||
|
|
my $status = sub {
|
||
|
|
my ($ct, $tot, $units, $user) = @_;
|
||
|
|
my $len = length($tot);
|
||
|
|
|
||
|
|
my $usertxt = $user ? " Moving user: $user" : '';
|
||
|
|
return sprintf(" \[%6.2f%%: %${len}d/%${len}d $units]$usertxt\n",
|
||
|
|
($ct / $tot) * 100, $ct, $tot);
|
||
|
|
};
|
||
|
|
|
||
|
|
my $header = sub {
|
||
|
|
my $size = 50;
|
||
|
|
return "\n" .
|
||
|
|
("#" x $size) . "\n" .
|
||
|
|
"# $_[0] " . (" " x ($size - length($_[0]) - 4)) . "#\n" .
|
||
|
|
("#" x $size) . "\n\n";
|
||
|
|
};
|
||
|
|
|
||
|
|
my $zeropad = sub {
|
||
|
|
return sprintf("%d", $_[0]);
|
||
|
|
};
|
||
|
|
|
||
|
|
# mover function
|
||
|
|
my $move_user = sub {
|
||
|
|
my $u = shift;
|
||
|
|
|
||
|
|
# make sure our user is of the proper dversion
|
||
|
|
return 0 unless $u->{'dversion'} == 6;
|
||
|
|
|
||
|
|
# at this point, try to get a lock for this user
|
||
|
|
my $lock;
|
||
|
|
if ($opts{locktype} eq 'ddlockd') {
|
||
|
|
$lock = LJ::locker()->trylock("d6d7-$u->{user}");
|
||
|
|
return 1 unless $lock;
|
||
|
|
}
|
||
|
|
|
||
|
|
# get a handle for every user to revalidate our connection?
|
||
|
|
my ($dbh, $dbcm) = $get_db_handles->($u->{clusterid});
|
||
|
|
return 0 unless $dbh;
|
||
|
|
|
||
|
|
# assign this dbcm to the user
|
||
|
|
if ($dbcm) {
|
||
|
|
$u->set_dbcm($dbcm)
|
||
|
|
or die "unable to set database for $u->{user}: dbcm=$dbcm\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
# verify dversion hasn't changed on us (done by another job?)
|
||
|
|
my $dversion = $dbh->selectrow_array("SELECT dversion FROM user WHERE userid = $u->{userid}");
|
||
|
|
return 1 unless $dversion == 6;
|
||
|
|
|
||
|
|
# ignore expunged users
|
||
|
|
if ($u->{'statusvis'} eq "X" || $u->{'clusterid'} == 0) {
|
||
|
|
LJ::update_user($u, { dversion => 7 })
|
||
|
|
or die "error updating dversion";
|
||
|
|
$u->{dversion} = 7; # update local copy in memory
|
||
|
|
return 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0 unless $dbcm;
|
||
|
|
|
||
|
|
# step 0.5: delete all the bogus userblob rows for this user
|
||
|
|
# This is due to the auto_increment for the blobid overflowing
|
||
|
|
# and thus all entries recieving an id of max id for a mediumint.
|
||
|
|
# This is lame.
|
||
|
|
my $domainid = LJ::get_blob_domainid('userpic');
|
||
|
|
$u->do("DELETE FROM userblob WHERE journalid=$u->{userid} AND domain=$domainid AND blobid>=16777216");
|
||
|
|
die "error in delete: " . $u->errstr . "\n" if $u->err;
|
||
|
|
|
||
|
|
# step 1: get all user pictures and move those. safe to just grab with no limit
|
||
|
|
# since users can only have a limited number of them
|
||
|
|
my $rows = $dbh->selectall_arrayref('SELECT picid, userid, contenttype, width, height, state, picdate, md5base64 ' .
|
||
|
|
'FROM userpic WHERE userid = ?', undef, $u->{userid}) || [];
|
||
|
|
|
||
|
|
if (@$rows) {
|
||
|
|
# got some rows, create an update statement
|
||
|
|
my (@bind, @vars, @blobids, @blobbind, @picinfo);
|
||
|
|
foreach my $row (@$rows) {
|
||
|
|
my $picid = $row->[0];
|
||
|
|
push @bind, "(?, ?, ?, ?, ?, ?, ?, ?)";
|
||
|
|
|
||
|
|
$row->[2] = {'image/gif' => 'G',
|
||
|
|
'image/jpeg' => 'J',
|
||
|
|
'image/png' => 'P'}->{$row->[2]};
|
||
|
|
push @vars, @$row;
|
||
|
|
|
||
|
|
# [picid, fmt]
|
||
|
|
my $fmt = {'G' => 'gif',
|
||
|
|
'J' => 'jpg',
|
||
|
|
'P' => 'png'}->{$row->[2]};
|
||
|
|
push @picinfo, [$picid, $fmt];
|
||
|
|
|
||
|
|
# picids
|
||
|
|
push @blobids, $picid;
|
||
|
|
push @blobbind, "?";
|
||
|
|
}
|
||
|
|
|
||
|
|
my $bind = join ',', @bind;
|
||
|
|
$u->do("REPLACE INTO userpic2 (picid, userid, fmt, width, height, state, picdate, md5base64) " .
|
||
|
|
"VALUES $bind", undef, @vars);
|
||
|
|
die "error in userpic2 replace: " . $u->errstr . "\n" if $u->err;
|
||
|
|
|
||
|
|
# step 1.5: insert missing rows into the userblob table
|
||
|
|
my $blobbind = join ',', @blobbind;
|
||
|
|
my $blobrows = $dbcm->selectall_hashref("SELECT blobid FROM userblob WHERE journalid=$u->{userid} AND domain=$domainid " .
|
||
|
|
"AND blobid IN ($blobbind)", 'blobid', undef, @blobids) || {};
|
||
|
|
|
||
|
|
my (@insertbind, @insertvars);
|
||
|
|
foreach my $pic (@picinfo) {
|
||
|
|
my ($picid, $fmt) = @$pic;
|
||
|
|
unless ($blobrows->{$picid}) {
|
||
|
|
push @insertbind, "(?, ?, ?, ?)";
|
||
|
|
|
||
|
|
my $blob = LJ::Blob::get($u, "userpic", $fmt, $picid);
|
||
|
|
my $length = length($blob);
|
||
|
|
|
||
|
|
push @insertvars, $u->{'userid'}, $domainid, $picid, $length;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (@insertbind) {
|
||
|
|
my $insertbind = join ',', @insertbind;
|
||
|
|
$u->do("INSERT IGNORE INTO userblob (journalid, domain, blobid, length) " .
|
||
|
|
"VALUES $insertbind", undef, @insertvars);
|
||
|
|
die "error in userblob insert: " . $u->errstr . "\n" if $u->err;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
# general purpose flusher for use below
|
||
|
|
my (@bind, @vars);
|
||
|
|
my $flush = sub {
|
||
|
|
return unless @bind;
|
||
|
|
my ($table, $cols) = @_;
|
||
|
|
|
||
|
|
# insert data into cluster master
|
||
|
|
my $bind = join(",", @bind);
|
||
|
|
$u->do("REPLACE INTO $table ($cols) VALUES $bind", undef, @vars);
|
||
|
|
die "error in flush $table: " . $u->errstr . "\n" if $u->err;
|
||
|
|
|
||
|
|
# reset values
|
||
|
|
@bind = ();
|
||
|
|
@vars = ();
|
||
|
|
};
|
||
|
|
|
||
|
|
# step 2: get the mapping of all of their keywords
|
||
|
|
my $kwrows = $dbh->selectall_arrayref('SELECT picid, kwid FROM userpicmap WHERE userid=?',
|
||
|
|
undef, $u->{'userid'});
|
||
|
|
my %kwmap;
|
||
|
|
if (@$kwrows) {
|
||
|
|
push @{$kwmap{$_->[1]}}, $_->[0] foreach @$kwrows; # kwid -> [ picid, picid, picid ... ]
|
||
|
|
}
|
||
|
|
|
||
|
|
# step 3: get the actual keywords associated with these keyword ids
|
||
|
|
my %kwidmap;
|
||
|
|
if (%kwmap) {
|
||
|
|
my $kwids = join ',', map { $_+0 } keys %kwmap;
|
||
|
|
my $rows = $dbh->selectall_arrayref("SELECT kwid, keyword FROM keywords WHERE kwid IN ($kwids)");
|
||
|
|
%kwidmap = map { $_->[0] => $_->[1] } @$rows; # kwid -> keyword
|
||
|
|
}
|
||
|
|
|
||
|
|
# step 4: now migrate all keywords into userkeywords table
|
||
|
|
my %mappings;
|
||
|
|
while (my ($kwid, $keyword) = each %kwidmap) {
|
||
|
|
# reallocate counter
|
||
|
|
my $newkwid = LJ::get_keyword_id($u, $keyword);
|
||
|
|
die "Error: unable to get keyword id for $u->{user}($u->{userid}), keyword '$keyword'\n"
|
||
|
|
unless $newkwid;
|
||
|
|
$mappings{$kwid} = $newkwid;
|
||
|
|
}
|
||
|
|
|
||
|
|
# step 5: now we have to do some mapping conversions and put new data into userpicmap2 table
|
||
|
|
while (my ($oldkwid, $picids) = each %kwmap) {
|
||
|
|
foreach my $picid (@$picids) {
|
||
|
|
# get new data
|
||
|
|
my $newkwid = $mappings{$oldkwid};
|
||
|
|
|
||
|
|
# push data
|
||
|
|
push @bind, "($u->{userid}, ?, ?)";
|
||
|
|
push @vars, ($picid, $newkwid);
|
||
|
|
|
||
|
|
# flush?
|
||
|
|
$flush->('userpicmap2', 'userid, picid, kwid')
|
||
|
|
if @bind > $BLOCK_INSERT;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
$flush->('userpicmap2', 'userid, picid, kwid');
|
||
|
|
|
||
|
|
# delete memcache keys that hold old data
|
||
|
|
LJ::MemCache::delete([$u->{userid}, "upicinf:$u->{userid}"]);
|
||
|
|
|
||
|
|
# haven't died yet? everything is still going okay, so update dversion
|
||
|
|
LJ::update_user($u, { 'dversion' => 7 })
|
||
|
|
or die "error updating dversion";
|
||
|
|
$u->{'dversion'} = 7; # update local copy in memory
|
||
|
|
|
||
|
|
return 1;
|
||
|
|
};
|
||
|
|
|
||
|
|
# get dbh handle
|
||
|
|
my $dbh = LJ::get_db_writer(); # just so we can get users...
|
||
|
|
die "Could not connect to global master" unless $dbh;
|
||
|
|
|
||
|
|
# get user count
|
||
|
|
my $total = $opts{total} || $dbh->selectrow_array("SELECT COUNT(*) FROM user WHERE dversion = 6");
|
||
|
|
$stats{'total_users'} = $total+0;
|
||
|
|
|
||
|
|
# print out header and total we're moving
|
||
|
|
print $header->("Moving user data");
|
||
|
|
print "Processing $stats{'total_users'} total users with the old dversion\n";
|
||
|
|
|
||
|
|
# loop until we have no more users to convert
|
||
|
|
my $ct;
|
||
|
|
while (1) {
|
||
|
|
# get users to move
|
||
|
|
my $sth;
|
||
|
|
if ($opts{user}) {
|
||
|
|
$sth = $dbh->prepare("SELECT * FROM user WHERE user = ? AND dversion = 6");
|
||
|
|
$sth->execute($opts{user});
|
||
|
|
} else {
|
||
|
|
$sth = $dbh->prepare("SELECT * FROM user WHERE dversion = 6 LIMIT $BLOCK_MOVE");
|
||
|
|
$sth->execute();
|
||
|
|
}
|
||
|
|
|
||
|
|
# get blocks of $BLOCK_MOVE users at a time
|
||
|
|
$ct = 0;
|
||
|
|
my (%us, %fast);
|
||
|
|
while (my $u = $sth->fetchrow_hashref()) {
|
||
|
|
$us{$u->{userid}} = $u;
|
||
|
|
$fast{$u->{userid}} = 1;
|
||
|
|
$ct++;
|
||
|
|
}
|
||
|
|
|
||
|
|
# jump out if we got nothing
|
||
|
|
last unless $ct;
|
||
|
|
|
||
|
|
# now that we have %us, we can see who has data
|
||
|
|
my $ids = join ',', map { $_+0 } keys %us;
|
||
|
|
my $has_upics = $dbh->selectcol_arrayref("SELECT DISTINCT userid FROM userpic WHERE userid IN ($ids)");
|
||
|
|
my %uids = ( map { $_ => 1 } (@$has_upics) );
|
||
|
|
|
||
|
|
# remove folks that have userpics from the fast list
|
||
|
|
delete $fast{$_} foreach keys %uids;
|
||
|
|
|
||
|
|
# now see who we can do in a fast way
|
||
|
|
my @fast_ids = map { $_+0 } keys %fast;
|
||
|
|
if (@fast_ids) {
|
||
|
|
# update stats for counting and print
|
||
|
|
$stats{'fast_moved'} += @fast_ids;
|
||
|
|
print $status->($stats{'slow_moved'}+$stats{'fast_moved'}, $stats{'total_users'}, "users");
|
||
|
|
|
||
|
|
# block update
|
||
|
|
LJ::update_user(\@fast_ids, { dversion => 7 });
|
||
|
|
}
|
||
|
|
|
||
|
|
my $slow_todo = scalar keys %uids;
|
||
|
|
print "Of $BLOCK_MOVE, $slow_todo have to be slow-converted...\n";
|
||
|
|
my @ids = randlist(keys %uids);
|
||
|
|
foreach my $id (@ids) {
|
||
|
|
# this person has userpics, move them the slow way
|
||
|
|
die "Userid $id in \$has_upics, but not in \%us...fatal error\n" unless $us{$id};
|
||
|
|
|
||
|
|
# now move the user
|
||
|
|
bless $us{$id}, 'LJ::User';
|
||
|
|
if ($move_user->($us{$id})) {
|
||
|
|
$stats{'slow_moved'}++;
|
||
|
|
print $status->($stats{'slow_moved'}+$stats{'fast_moved'}, $stats{'total_users'}, "users", $us{$id}{user});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
# ...done?
|
||
|
|
print $header->("Dversion 6->7 conversion completed");
|
||
|
|
print " Users moved: " . $zeropad->($stats{'slow_moved'}) . "\n";
|
||
|
|
print "Users updated: " . $zeropad->($stats{'fast_moved'}) . "\n\n";
|
||
|
|
|
||
|
|
# helper function to randomize stuff
|
||
|
|
sub randlist
|
||
|
|
{
|
||
|
|
my @rlist = @_;
|
||
|
|
my $size = scalar(@rlist);
|
||
|
|
|
||
|
|
my $i;
|
||
|
|
for ($i=0; $i<$size; $i++) {
|
||
|
|
unshift @rlist, splice(@rlist, $i+int(rand()*($size-$i)), 1);
|
||
|
|
}
|
||
|
|
return @rlist;
|
||
|
|
}
|