zlax/ljr
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

init

Browse Source
This commit is contained in:
ivan
2019-02-06 00:49:12 +03:00
commit 8dbb1bb605
4796 changed files with 506072 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
wcmtools/spud/bin/plugins/config_generator.pl Executable file
View File

@@ -0,0 +1,113 @@
# automatic monitoring config generator plugin for LiveJournal. to use, add a
# line like this to your SPUD config:
#
# config_generator(mysql, perlbal, memcached, mogstored, mogilefsd)
#
# written by Mark Smith <junior@danga.com>
package ConfigGenPlugin;
use strict;
# called when we're loaded. here we can do anything necessary to set ourselves
# up if we want. in this case we just load the LJ libraries.
sub register {
# load up our livejournal files
use lib "$ENV{LJHOME}/cgi-bin";
require 'ljlib.pl';
# signal success if we get here
return 1;
}
# this is called and given the job name as the first parameter and an array ref of
# options passed in as the second parameter.
sub helper {
my $options = shift;
# put options into hashref for easy use later
my %opts;
foreach my $opt (@$options) {
my @parms = split(/\s*=\s*/, $opt);
my $job = shift(@parms);
$opts{$job} = \@parms;
}
# this is the main loop
LJ::start_request();
# mark all of our jobs as being inactive so that if we don't readd them below
# they'll get reaped automatically.
mark_inactive_by_plugin('config_generator');
# look for any perlbals that need monitoring jobs
if ($opts{perlbal}) {
while (my ($srvr, $ipaddr) = each %LJ::PERLBAL_SERVERS) {
add_job("perlbal.$srvr", "perlbal", [ $ipaddr, @{$opts{perlbal}} ], 'config_generator');
}
}
# and now memcache servers
if ($opts{memcached}) {
foreach my $host (@LJ::MEMCACHE_SERVERS) {
my $ipaddr = ref $host ? $host->[0] : $host;
add_job("memcached.$ipaddr", "memcached", [ $ipaddr, @{$opts{memcached}} ], 'config_generator');
}
}
# mogilefsd
if ($opts{mogilefsd} && %LJ::MOGILEFS_CONFIG) {
foreach my $ipaddr (@{$LJ::MOGILEFS_CONFIG{hosts}}) {
add_job("mogilefsd.$ipaddr", "mogilefsd", [ $ipaddr, @{$opts{mogilefsd}} ], 'config_generator');
}
}
# mogstored
if ($opts{mogstored} && %LJ::MOGILEFS_CONFIG) {
my $mgd = new MogileFS::Admin(hosts => $LJ::MOGILEFS_CONFIG{hosts});
if ($mgd) {
my (%hosthash, %devhash);
if (my $hosts = $mgd->get_hosts) {
foreach my $h (@$hosts) {
$hosthash{$h->{hostid}} = $h;
}
}
if (my $devs = $mgd->get_devices) {
foreach my $d (@$devs) {
$devhash{$d->{devid}} = $d;
}
}
foreach my $devid (keys %devhash) {
my $host = $hosthash{$devhash{$devid}->{hostid}};
add_job("mogstored.dev$devid", "mogstored",
[ "http://$host->{hostip}:$host->{http_port}/dev$devid/usage", @{$opts{mogstored}} ],
'config_generator');
}
foreach my $host (values %hosthash) {
my $ipaddr = "$host->{hostip}:7501";
add_job("mogstored.$ipaddr", "perlbal", [ $ipaddr, @{$opts{perlbal} || []} ], 'config_generator');
}
}
}
if ($opts{mysql} || $opts{db} || $opts{database}) {
}
# done, call end request and sleep for a while
LJ::end_request();
}
# calls the registrar in the main program, giving them information about us. this
# has to be called as main:: or just ::register_plugin because we're in our own
# package and we want to talk to the register function in the main namespace.
main::register_plugin('config_generator', 'ConfigGenPlugin', {
register => \&register,
helper => \&helper,
});
1;

63
wcmtools/spud/bin/plugins/memcached.pl Executable file
View File

@@ -0,0 +1,63 @@
# memcache monitoring plugin for SPUD. this is a simple plugin that gets stats
# information from memcache and sticks it in the server.
#
# written by Mark Smith <junior@danga.com>
package MemcachedPlugin;
use strict;
# called when we're loaded. here we can do anything necessary to set ourselves
# up if we want.
sub register {
debug("memcached plugin registered");
return 1;
}
# this is called and given the job name as the first parameter and an array ref of
# options passed in as the second parameter.
sub worker {
my ($job, $options) = @_;
my $ipaddr = shift(@{$options || []});
my $interval = shift(@{$options || []}) || 5;
return unless $ipaddr;
# loop and get statistics every second
my $sock;
my $read_input = sub {
my @out;
while (<$sock>) {
s/[\r\n\s]+$//;
last if /^END/;
push @out, $_;
}
return \@out;
};
while (1) {
$sock ||= IO::Socket::INET->new(PeerAddr => $ipaddr, Timeout => 3);
return unless $sock;
# basic states command
print $sock "stats\r\n";
my $out = $read_input->();
foreach my $line (@$out) {
if ($line =~ /^STAT\s+([\w:]+)\s+(.+)$/) {
my ($stat, $val) = ($1, $2);
set("$job.$stat", $val);
}
}
# now sleep some between doing things
sleep $interval;
}
}
# calls the registrar in the main program, giving them information about us. this
# has to be called as main:: or just ::register_plugin because we're in our own
# package and we want to talk to the register function in the main namespace.
main::register_plugin('memcached', 'MemcachedPlugin', {
register => \&register,
worker => \&worker,
});
1;

64
wcmtools/spud/bin/plugins/mogilefsd.pl Executable file
View File

@@ -0,0 +1,64 @@
# mogilefsd monitoring plugin. this looks at the stats which is a very quick
# operation for the mogilefsd server. plans for this plugin are to start
# monitoring replication, recent queries, etc.
#
# written by Mark Smith <junior@danga.com>
package MogilefsdPlugin;
use strict;
# called when we're loaded. here we can do anything necessary to set ourselves
# up if we want.
sub register {
debug("mogilefsd plugin registered");
return 1;
}
# this is called and given the job name as the first parameter and an array ref of
# options passed in as the second parameter.
sub worker {
my ($job, $options) = @_;
my $ipaddr = shift(@{$options || []});
my $interval = shift(@{$options || []}) || 5;
return unless $ipaddr;
# test plugin simply loops and once a second sets a "heartbeat"
my $sock;
my $read_input = sub {
my @out;
while (<$sock>) {
s/[\r\n\s]+$//;
last if /^\./;
push @out, $_;
}
return \@out;
};
while (1) {
$sock ||= IO::Socket::INET->new(PeerAddr => $ipaddr, Timeout => 3);
return unless $sock;
# basic states command
print $sock "!stats\r\n";
my $out = $read_input->();
foreach my $line (@$out) {
if ($line =~ /^([\w:]+)\s+(.+)$/) {
my ($stat, $val) = ($1, $2);
set("$job.$stat", $val);
}
}
# now sleep some between doing things
sleep $interval;
}
}
# calls the registrar in the main program, giving them information about us. this
# has to be called as main:: or just ::register_plugin because we're in our own
# package and we want to talk to the register function in the main namespace.
main::register_plugin('mogilefsd', 'MogilefsdPlugin', {
register => \&register,
worker => \&worker,
});
1;

74
wcmtools/spud/bin/plugins/mogstored.pl Executable file
View File

@@ -0,0 +1,74 @@
# mogstored device monitoring plugin. this fetches the 'usage' file from a device
# and parses it, putting the information in the server.
#
# written by Mark Smith <junior@danga.com>
package MogstoredPlugin;
# packages we need
use LWP::Simple;
use Time::HiRes qw(gettimeofday tv_interval);
# called when we're loaded. here we can do anything necessary to set ourselves
# up if we want.
sub register {
debug("mogstored plugin registered");
return 1;
}
# this is called and given the job name as the first parameter and an array ref of
# options passed in as the second parameter.
sub worker {
my ($job, $options) = @_;
my $url = shift(@{$options || []});
my $interval = shift(@{$options || []}) || 60;
return unless $url;
# get stats every $interval seconds
while (1) {
my $t0 = [ gettimeofday ];
my $doc = get($url);
my $time = tv_interval($t0);
unless (defined $doc) {
set("$job.status", "fetch_failure");
sleep $interval;
next;
}
# split the doc and parse
my %stats;
foreach (split(/\r?\n/, $doc)) {
next unless /^(\w+):\s+(.+)$/;
my ($key, $val) = ($1, $2);
$stats{$key} = $val;
}
# if we couldn't parse it
unless ($stats{time} && $stats{total} && $stats{used} && $stats{available}) {
set("$job.status", "parse_failure");
sleep $interval;
next;
}
# mark this as successfully retrieved
set("$job.status", "success");
set("$job.time", $stats{time});
set("$job.used", $stats{used});
set("$job.available", $stats{available});
set("$job.total", $stats{total});
set("$job.delay", sprintf("%5.3f", $time));
# sleep a good 60 seconds, as this file doesn't change very often
sleep $interval;
}
}
# calls the registrar in the main program, giving them information about us. this
# has to be called as main:: or just ::register_plugin because we're in our own
# package and we want to talk to the register function in the main namespace.
main::register_plugin('mogstored', 'MogstoredPlugin', {
register => \&register,
worker => \&worker,
});
1;

65
wcmtools/spud/bin/plugins/perlbal.pl Executable file
View File

@@ -0,0 +1,65 @@
# perlbal monitoring plugin. very simple right now, this gets the output of the states
# command and saves it to spud. this is also used on the mogstored sidechannel, which
# is a perlbal management interface.
#
# written by Mark Smith <junior@danga.com>
package PerlbalPlugin;
use strict;
# called when we're loaded. here we can do anything necessary to set ourselves
# up if we want.
sub register {
debug("perlbal plugin registered");
return 1;
}
# this is called and given the job name as the first parameter and an array ref of
# options passed in as the second parameter.
sub worker {
my ($job, $options) = @_;
my $ipaddr = shift(@{$options || []});
my $interval = shift(@{$options || []}) || 5;
return unless $ipaddr;
# try to get states every second
my $sock;
my $read_input = sub {
my @out;
while (<$sock>) {
s/[\r\n\s]+$//;
last if /^\./;
push @out, $_;
}
return \@out;
};
while (1) {
$sock ||= IO::Socket::INET->new(PeerAddr => $ipaddr, Timeout => 3);
return unless $sock;
# basic states command
print $sock "states\r\n";
my $out = $read_input->();
foreach my $line (@$out) {
if ($line =~ /^(.+?)\s+(\w+)\s+(\d+)$/) {
my ($class, $state, $count) = ($1, $2, $3);
$class =~ s/^(.+::)//;
set("$job.$class.$state", $count);
}
}
# now sleep some between doing things
sleep $interval;
}
}
# calls the registrar in the main program, giving them information about us. this
# has to be called as main:: or just ::register_plugin because we're in our own
# package and we want to talk to the register function in the main namespace.
main::register_plugin('perlbal', 'PerlbalPlugin', {
register => \&register,
worker => \&worker,
});
1;

37
wcmtools/spud/bin/plugins/test.pl Executable file
View File

@@ -0,0 +1,37 @@
# test/heartbeat plugin for SPUD statistic gathering system
# written by Mark Smith <junior@danga.com>
# this is mostly a demonstration of how to create a very simple plugin for SPUD.
# more complex examples can be found elsewhere in the plugins directory.
# doesn't matter what package you're in
package TestPlugin;
# called when we're loaded. here we can do anything necessary to set ourselves
# up if we want.
sub register {
debug("test plugin registered");
return 1;
}
# this is called and given the job name as the first parameter and an array ref of
# options passed in as the second parameter.
sub worker {
my ($job, $options) = @_;
# test plugin simply loops and once a second sets a "heartbeat"
while (1) {
set("test.$job" => 1);
sleep 1;
}
}
# calls the registrar in the main program, giving them information about us. this
# has to be called as main:: or just ::register_plugin because we're in our own
# package and we want to talk to the register function in the main namespace.
main::register_plugin('test', 'TestPlugin', {
register => \&register,
worker => \&worker,
});
1;