From 6c82a76b912c7a15ab58b339e7dfb511687846fa Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 23 Dec 2007 08:08:30 +0000 Subject: initial perl-based client and server with shiny IPC svn path=/branches/new-fu/; revision=252 --- client/scireclient.pl | 23 +++++++++++++++++++++++ server/scireserver.pl | 20 ++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100755 client/scireclient.pl create mode 100755 server/scireserver.pl diff --git a/client/scireclient.pl b/client/scireclient.pl new file mode 100755 index 0000000..abf5f3f --- /dev/null +++ b/client/scireclient.pl @@ -0,0 +1,23 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use IPC::Open2; + +my $jobdir = "/tmp/scirejobs"; +my $servercmd = "../server/scireserver.pl"; + +if(! -d $jobdir) { + print "WARNING! ${jobdir} does not exist...creating\n"; + mkdir $jobdir, 0660; +} + +my $pid = open2(*server_stdout, *server_stdin, $servercmd); + +for(('PING', 'FOO', 'QUIT')) { + print "Sending: $_\n"; + print server_stdin "$_\n"; + my $response = ; + print "Got: ${response}"; +} diff --git a/server/scireserver.pl b/server/scireserver.pl new file mode 100755 index 0000000..e328118 --- /dev/null +++ b/server/scireserver.pl @@ -0,0 +1,20 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +$| = 1; +my $jobdir = "/tmp/scirejobs"; + +while(<>) { + my $line = $_; + chomp $line; + if($line =~ /^PING$/) { + print "PONG\n"; + } elsif($line =~ /^QUIT$/) { + print "Exiting!\n"; + exit; + } else { + print "Unknown command\n"; + } +} -- cgit v1.2.3-65-gdbad From 46fe9eb463409d5bcd9f6b8bea6f1e8ec2fce5c1 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 23 Dec 2007 18:02:22 +0000 Subject: add servercmd comment svn path=/branches/new-fu/; revision=253 --- client/scireclient.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/client/scireclient.pl b/client/scireclient.pl index abf5f3f..9bf7aed 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -6,6 +6,7 @@ use warnings; use IPC::Open2; my $jobdir = "/tmp/scirejobs"; +# This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" my $servercmd = "../server/scireserver.pl"; if(! -d $jobdir) { -- cgit v1.2.3-65-gdbad From 759cbf7249ff7871567efb3e261f7098c4a81bde Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 24 Dec 2007 00:21:22 +0000 Subject: adding config-related stuff. svn path=/branches/new-fu/; revision=254 --- client/scireclient.pl | 45 +++++++++++++++++++++++++++++++++++++++------ etc/scire.conf | 5 +++++ 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 etc/scire.conf diff --git a/client/scireclient.pl b/client/scireclient.pl index 9bf7aed..e1f287c 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -5,20 +5,53 @@ use warnings; use IPC::Open2; -my $jobdir = "/tmp/scirejobs"; +use constant SCIRE_CONFIG_FILE => '/etc/scire.conf'; +#Constants to be defined in /etc/scire.conf + +my $conf; +read_config(); + +#Read command line options. + +# Build the connection command. # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" -my $servercmd = "../server/scireserver.pl"; +my $connection_command = "ssh "; +if (defined($conf{port})) { + $connection_command .= "-o Port=$conf{port} "; +} +$connection_command .= "$conf{user}\@$conf{host} $conf{server_script}"; -if(! -d $jobdir) { - print "WARNING! ${jobdir} does not exist...creating\n"; - mkdir $jobdir, 0660; +#my $servercmd = "../server/scireserver.pl"; + +if(! -d $conf{job_dir}) { + print "WARNING! $conf{job_dir} does not exist...creating\n"; + mkdir $conf{job_dir}, 0660; } -my $pid = open2(*server_stdout, *server_stdin, $servercmd); +my $pid = open2(*server_stdout, *server_stdin, $connection_command); +sub run_main { + #ok folks so here's how this thang goes down. + #1. Connect. + #2. Register with the DB. (only it knows if you're allowed to be active) + #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. + #4. Fetch the jobs list + #5. ? + # for(('PING', 'FOO', 'QUIT')) { print "Sending: $_\n"; print server_stdin "$_\n"; my $response = ; print "Got: ${response}"; } + +sub read_config { + my FH = open(SCIRE_CONFIG_FILE) or die("Couldn't open the config ".${\SCIRE_CONFIG_FILE}." : $!"); + while () { + chomp( my $line = $_); + next if /^\s*(?:#|$)/; + my ($key,$val) = split /=/; + $conf{lc($key)} = $val; + } + close(FH) or die("Couldn't close the config ".${\SCIRE_CONFIG_FILE}." : $!"); +} diff --git a/etc/scire.conf b/etc/scire.conf new file mode 100644 index 0000000..e205453 --- /dev/null +++ b/etc/scire.conf @@ -0,0 +1,5 @@ +HOST=localhost +PORT=22 +USERNAME=scire +SERVER_SCRIPT=../server/scireserver.pl +JOB_DIR=/tmp/scirejobs -- cgit v1.2.3-65-gdbad From dccea1b1d5eaa006f04e9851d2841159f6de96b5 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 01:28:00 +0000 Subject: change var name from USERNAME to USER to match existing code svn path=/branches/new-fu/; revision=255 --- etc/scire.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/scire.conf b/etc/scire.conf index e205453..578885f 100644 --- a/etc/scire.conf +++ b/etc/scire.conf @@ -1,5 +1,5 @@ HOST=localhost PORT=22 -USERNAME=scire +USER=scire SERVER_SCRIPT=../server/scireserver.pl JOB_DIR=/tmp/scirejobs -- cgit v1.2.3-65-gdbad From 8e080c2d7e1d86fc5ef3bd4c48ac91f789db05dd Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 01:30:35 +0000 Subject: ok, where to begin.... SCIRE_CONFIG_FILE is no longer a constant so it can be overriden via commandline basic commandline parsing helper function for sending/receiving and creating the connection move my test code into run_test() function temporary connection_command override for development fix a few syntax errors random other changes that I can't be bothered to think of svn path=/branches/new-fu/; revision=256 --- client/scireclient.pl | 80 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index e1f287c..3e03049 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -5,30 +5,32 @@ use warnings; use IPC::Open2; -use constant SCIRE_CONFIG_FILE => '/etc/scire.conf'; -#Constants to be defined in /etc/scire.conf +my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; +my %conf; +my ($SERVER_STDOUT, $SERVER_STDIN); -my $conf; +parse_command_line(); read_config(); -#Read command line options. - # Build the connection command. # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" my $connection_command = "ssh "; -if (defined($conf{port})) { +if(defined($conf{port})) { $connection_command .= "-o Port=$conf{port} "; } $connection_command .= "$conf{user}\@$conf{host} $conf{server_script}"; -#my $servercmd = "../server/scireserver.pl"; +if(-d ".svn") { + # Overwrite $connection_command in the case of a dev environment for now + $connection_command = "../server/scireserver.pl"; +} if(! -d $conf{job_dir}) { print "WARNING! $conf{job_dir} does not exist...creating\n"; mkdir $conf{job_dir}, 0660; } -my $pid = open2(*server_stdout, *server_stdin, $connection_command); +run_main(); sub run_main { #ok folks so here's how this thang goes down. @@ -37,21 +39,65 @@ sub run_main { #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. #4. Fetch the jobs list #5. ? - # -for(('PING', 'FOO', 'QUIT')) { - print "Sending: $_\n"; - print server_stdin "$_\n"; - my $response = ; - print "Got: ${response}"; + create_connection(); + run_test(); +} + +sub run_test { + for(('PING', 'FOO', 'QUIT')) { + send_command($_); + my $response = get_response(); + print "Got: ${response}"; + } +} + +sub parse_command_line { + # XXX: Anyone know how to use getopt? I've never bothered figuring it out :P + while($#ARGV > -1) { + my $foo = shift @ARGV; + if($foo eq "-f") { + $SCIRE_CONFIG_FILE = shift @ARGV; + } else { + print "Unrecognized command line option: ${foo}\n"; + } + } +} + +sub send_command { + my $cmd = shift; + my @args = @_; + my $tosend = "${cmd}"; + + for my $arg (@args) { + if($arg =~ /^[0-9]+$/) { + $tosend .= " ${arg}"; + } else { + $arg =~ s/"/\\"/g; + $tosend .= " \"${arg}\""; + } + } + print "Sending: ${tosend}\n"; + print SERVER_STDIN "${tosend}\n"; +} + +sub get_response { + # XXX: Add some logic for multi-line responses here + my $response = ; + return $response; +} + +sub create_connection { + # XXX: We need to see what this function returns if the command returns non-zero + my $pid = open2(*SERVER_STDOUT, *SERVER_STDIN, $connection_command); } sub read_config { - my FH = open(SCIRE_CONFIG_FILE) or die("Couldn't open the config ".${\SCIRE_CONFIG_FILE}." : $!"); + open(FH, "< ${SCIRE_CONFIG_FILE}") or die("Couldn't open the config file ${SCIRE_CONFIG_FILE}: $!"); while () { - chomp( my $line = $_); + chomp; next if /^\s*(?:#|$)/; my ($key,$val) = split /=/; $conf{lc($key)} = $val; } - close(FH) or die("Couldn't close the config ".${\SCIRE_CONFIG_FILE}." : $!"); + close(FH) or die("Couldn't close the config file ${SCIRE_CONFIG_FILE}: $!"); } -- cgit v1.2.3-65-gdbad From c57661aed688963c59cea8ed1429381d1c73dac0 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 02:29:48 +0000 Subject: use a match and capture instead of split for parsing config file lines svn path=/branches/new-fu/; revision=257 --- client/scireclient.pl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 3e03049..47fd7b9 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -96,8 +96,9 @@ sub read_config { while () { chomp; next if /^\s*(?:#|$)/; - my ($key,$val) = split /=/; - $conf{lc($key)} = $val; + if(/^\s*(.+?)\s*=\s*(.+?)\s*(?:#.*)?$/) { + $conf{lc($1)} = $2; + } } close(FH) or die("Couldn't close the config file ${SCIRE_CONFIG_FILE}: $!"); } -- cgit v1.2.3-65-gdbad From 92b34343284e0cd742fa8f6e22a1ba7a20ad37b9 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 02:30:05 +0000 Subject: toss in a comment...just because svn path=/branches/new-fu/; revision=258 --- etc/scire.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/scire.conf b/etc/scire.conf index 578885f..5d01d27 100644 --- a/etc/scire.conf +++ b/etc/scire.conf @@ -2,4 +2,4 @@ HOST=localhost PORT=22 USER=scire SERVER_SCRIPT=../server/scireserver.pl -JOB_DIR=/tmp/scirejobs +JOB_DIR=/tmp/scirejobs # Comments at the end of the line are a-okay -- cgit v1.2.3-65-gdbad From 4b32204d45762c527ac1266e5314c0eb66d48e18 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 24 Dec 2007 06:27:16 +0000 Subject: config touchups. added a register function. it needs a fingerprint or digest to verify the client. svn path=/branches/new-fu/; revision=259 --- client/scireclient.pl | 74 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 47fd7b9..d027d59 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -1,16 +1,21 @@ #!/usr/bin/perl +# $Id$ + use strict; use warnings; use IPC::Open2; +use Getopt::Long; +use Data::Dumper; -my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; +my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; my ($SERVER_STDOUT, $SERVER_STDIN); parse_command_line(); -read_config(); +my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; +read_config($conf_file); # Build the connection command. # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" @@ -20,14 +25,15 @@ if(defined($conf{port})) { } $connection_command .= "$conf{user}\@$conf{host} $conf{server_script}"; -if(-d ".svn") { +if (-d ".svn") { # Overwrite $connection_command in the case of a dev environment for now $connection_command = "../server/scireserver.pl"; } -if(! -d $conf{job_dir}) { - print "WARNING! $conf{job_dir} does not exist...creating\n"; - mkdir $conf{job_dir}, 0660; +if (! -d $conf{job_dir}) { + print "WARNING! $conf{job_dir} does not exist...creating\n"; + mkdir -p $conf{job_dir}, 0660 + or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); } run_main(); @@ -35,11 +41,14 @@ run_main(); sub run_main { #ok folks so here's how this thang goes down. #1. Connect. + create_connection(); + #2. Register with the DB. (only it knows if you're allowed to be active) + register_client(); + #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. #4. Fetch the jobs list #5. ? - create_connection(); run_test(); } @@ -52,15 +61,28 @@ sub run_test { } sub parse_command_line { - # XXX: Anyone know how to use getopt? I've never bothered figuring it out :P - while($#ARGV > -1) { - my $foo = shift @ARGV; - if($foo eq "-f") { - $SCIRE_CONFIG_FILE = shift @ARGV; - } else { - print "Unrecognized command line option: ${foo}\n"; - } + GetOptions( + 'verbose|D' => \$conf{debug}, + 'dry-run' => \$conf{dry_run}, + 'help|h' => \$conf{help}, + 'config|c=s' => \$conf{config}, + 'threads|t=i' => \$conf{max_threads}, + + #config overrides. + 'host=s' => \$conf{host}, + 'port=i' => \$conf{port}, + 'user|u=s' => \$conf{user}, + 'server_script=s' => \$conf{server_script}, + 'job_dir' => \$conf{job_dir}, + ); + if ($conf{help}) { + print "\nusage: scireclient.pl [--verbose or -D]\n\t [--dry-run]" + ."\t [--config=CONF or -c] \n\t [--threads=# or -t] \t [--help or -h] \n" + ."\t [[--host=HOST] \t [--port=PORT] \t [--user=USER or -u] \n\t" + ." [--server_script=foo.pl] \t [--job_dir=/tmp/jobs] \n"; + exit 0; } + } sub send_command { @@ -76,7 +98,7 @@ sub send_command { $tosend .= " \"${arg}\""; } } - print "Sending: ${tosend}\n"; + print "Sending: ${tosend}\n" if $conf{debug}; print SERVER_STDIN "${tosend}\n"; } @@ -92,13 +114,27 @@ sub create_connection { } sub read_config { - open(FH, "< ${SCIRE_CONFIG_FILE}") or die("Couldn't open the config file ${SCIRE_CONFIG_FILE}: $!"); + my $conf_file = shift; + open(FH, "< ${conf_file}") or die("Couldn't open the config file ${conf_file}: $!"); while () { chomp; next if /^\s*(?:#|$)/; if(/^\s*(.+?)\s*=\s*(.+?)\s*(?:#.*)?$/) { - $conf{lc($1)} = $2; + unless(defined($conf{lc($1)})) { #Don't overwrite anything specified in cmdline + $conf{lc($1)} = $2; + } } } - close(FH) or die("Couldn't close the config file ${SCIRE_CONFIG_FILE}: $!"); + close(FH) or die("Couldn't close the config file ${conf_file}: $!"); +} + +sub register_client { + my $fingerprint = "AA:BB:CC:DD:EE:FF:GG:HH... this is a digest."; + my $cmd = "IDENTIFY $fingerprint"; + send_command($cmd); + my $response = get_response(); + $response =~ /IDENTIFIED\ CLIENT\ (\d+)/ + or die("Couldn't register client. Response was $response"); + $conf{client_id} = $1; + print "Registered client $conf{client_id}\n" if $conf{debug}; } -- cgit v1.2.3-65-gdbad From 53fae43967313c5e025a40b5bfea429d2c76e752 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 07:18:23 +0000 Subject: rename verbose to debug split identify and register svn path=/branches/new-fu/; revision=260 --- client/scireclient.pl | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index d027d59..bf47e43 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -32,7 +32,7 @@ if (-d ".svn") { if (! -d $conf{job_dir}) { print "WARNING! $conf{job_dir} does not exist...creating\n"; - mkdir -p $conf{job_dir}, 0660 + mkdir $conf{job_dir}, 0660 or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); } @@ -44,7 +44,11 @@ sub run_main { create_connection(); #2. Register with the DB. (only it knows if you're allowed to be active) - register_client(); + if(-f "../etc/client_key") { + identify_client(); + } else { +# register_client(); + } #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. #4. Fetch the jobs list @@ -62,7 +66,7 @@ sub run_test { sub parse_command_line { GetOptions( - 'verbose|D' => \$conf{debug}, + 'debug|D' => \$conf{debug}, 'dry-run' => \$conf{dry_run}, 'help|h' => \$conf{help}, 'config|c=s' => \$conf{config}, @@ -76,7 +80,7 @@ sub parse_command_line { 'job_dir' => \$conf{job_dir}, ); if ($conf{help}) { - print "\nusage: scireclient.pl [--verbose or -D]\n\t [--dry-run]" + print "\nusage: scireclient.pl [--debug or -D]\n\t [--dry-run]" ."\t [--config=CONF or -c] \n\t [--threads=# or -t] \t [--help or -h] \n" ."\t [[--host=HOST] \t [--port=PORT] \t [--user=USER or -u] \n\t" ." [--server_script=foo.pl] \t [--job_dir=/tmp/jobs] \n"; @@ -109,7 +113,12 @@ sub get_response { } sub create_connection { - # XXX: We need to see what this function returns if the command returns non-zero + # XXX: How do we capture this error? $pid has a valid value even if the + # process fails to run, since it just returns the PID of the forked perl + # process. I tried adding 'or die' after it, but it didn't help since it + # doesn't fail in the main process. When it fails, it outputs an error + # to STDERR: + # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 my $pid = open2(*SERVER_STDOUT, *SERVER_STDIN, $connection_command); } @@ -129,12 +138,19 @@ sub read_config { } sub register_client { - my $fingerprint = "AA:BB:CC:DD:EE:FF:GG:HH... this is a digest."; - my $cmd = "IDENTIFY $fingerprint"; + send_command("REGISTER 00:11:22:33:44:55 192.168.2.3 myhostname"); +} + +sub identify_client { + open FILE, "< ../etc/client_key" or die "Couldn't open client_key: $!"; + my $client_key = join("", ); + close FILE; + my $cmd = "IDENTIFY ${client_key}"; send_command($cmd); my $response = get_response(); - $response =~ /IDENTIFIED\ CLIENT\ (\d+)/ - or die("Couldn't register client. Response was $response"); - $conf{client_id} = $1; + $response =~ /^(OK|ERROR (.+))$/; + if($1 eq "ERROR") { + print "Could not identify to server: $2\n"; + } print "Registered client $conf{client_id}\n" if $conf{debug}; } -- cgit v1.2.3-65-gdbad From cff09eeeed71a7b56fcaac1176e557fda9100d41 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 24 Dec 2007 15:56:33 +0000 Subject: musta been too drunk to not remember mkpath. svn path=/branches/new-fu/; revision=261 --- client/scireclient.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index bf47e43..a1708f0 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -8,6 +8,7 @@ use warnings; use IPC::Open2; use Getopt::Long; use Data::Dumper; +use File::Path; my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; @@ -32,7 +33,7 @@ if (-d ".svn") { if (! -d $conf{job_dir}) { print "WARNING! $conf{job_dir} does not exist...creating\n"; - mkdir $conf{job_dir}, 0660 + mkpath( $conf{job_dir}, {verbose => 1, mode => 0660}) or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); } -- cgit v1.2.3-65-gdbad From 6913310a84b72d913eead96e6e04dbbe1ff977f0 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 18:29:14 +0000 Subject: implement IDENTIFY command with random response in server svn path=/branches/new-fu/; revision=262 --- server/scireserver.pl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index e328118..2c7a718 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -9,12 +9,19 @@ my $jobdir = "/tmp/scirejobs"; while(<>) { my $line = $_; chomp $line; - if($line =~ /^PING$/) { - print "PONG\n"; + if($line =~ /^IDENTIFY (.+)$/) { + my $rand_int = int(rand(3)); + if($rand_int == 0) { + print "OK\n"; + } elsif($rand_int == 1) { + print "ERROR Unrecognized client key. Please register\n"; + } elsif($rand_int == 2) { + print "ERROR This client has not yet been authorized\n"; + } } elsif($line =~ /^QUIT$/) { - print "Exiting!\n"; + print "OK\n"; exit; } else { - print "Unknown command\n"; + print "ERROR Unknown command\n"; } } -- cgit v1.2.3-65-gdbad From bb59574e35600f77f989458ceb68bba2a1c9fca3 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 24 Dec 2007 18:30:59 +0000 Subject: move connection string building into its own function move most of global stuff inside run_main react properly to responses from IDENTIFY command move the job_dir check into its own function svn path=/branches/new-fu/; revision=263 --- client/scireclient.pl | 90 +++++++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index a1708f0..fc02c54 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -14,42 +14,29 @@ my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when relea my %conf; my ($SERVER_STDOUT, $SERVER_STDIN); -parse_command_line(); -my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; -read_config($conf_file); - -# Build the connection command. -# This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" -my $connection_command = "ssh "; -if(defined($conf{port})) { - $connection_command .= "-o Port=$conf{port} "; -} -$connection_command .= "$conf{user}\@$conf{host} $conf{server_script}"; +run_main(); -if (-d ".svn") { - # Overwrite $connection_command in the case of a dev environment for now - $connection_command = "../server/scireserver.pl"; -} +sub run_main { + parse_command_line(); + my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; + read_config_file($conf_file); -if (! -d $conf{job_dir}) { - print "WARNING! $conf{job_dir} does not exist...creating\n"; - mkpath( $conf{job_dir}, {verbose => 1, mode => 0660}) - or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); -} + check_job_dir(); -run_main(); + my $connection_command = build_connection_command(); -sub run_main { #ok folks so here's how this thang goes down. #1. Connect. - create_connection(); + create_connection($connection_command); #2. Register with the DB. (only it knows if you're allowed to be active) - if(-f "../etc/client_key") { - identify_client(); - } else { +# if(-f "../etc/client_key") { + if(!identify_client()) { + exit(1); + } +# } else { # register_client(); - } +# } #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. #4. Fetch the jobs list @@ -61,13 +48,12 @@ sub run_test { for(('PING', 'FOO', 'QUIT')) { send_command($_); my $response = get_response(); - print "Got: ${response}"; } } sub parse_command_line { GetOptions( - 'debug|D' => \$conf{debug}, + 'debug|d' => \$conf{debug}, 'dry-run' => \$conf{dry_run}, 'help|h' => \$conf{help}, 'config|c=s' => \$conf{config}, @@ -81,7 +67,7 @@ sub parse_command_line { 'job_dir' => \$conf{job_dir}, ); if ($conf{help}) { - print "\nusage: scireclient.pl [--debug or -D]\n\t [--dry-run]" + print "\nusage: scireclient.pl [--debug or -d]\n\t [--dry-run]" ."\t [--config=CONF or -c] \n\t [--threads=# or -t] \t [--help or -h] \n" ."\t [[--host=HOST] \t [--port=PORT] \t [--user=USER or -u] \n\t" ." [--server_script=foo.pl] \t [--job_dir=/tmp/jobs] \n"; @@ -110,6 +96,7 @@ sub send_command { sub get_response { # XXX: Add some logic for multi-line responses here my $response = ; + print "Got: ${response}" if($conf{debug}); return $response; } @@ -120,10 +107,35 @@ sub create_connection { # doesn't fail in the main process. When it fails, it outputs an error # to STDERR: # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 + my $connection_command = shift; my $pid = open2(*SERVER_STDOUT, *SERVER_STDIN, $connection_command); } -sub read_config { +sub build_connection_command { + # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" + my $connection_command = "ssh "; + if(defined($conf{port})) { + $connection_command .= "-o Port=$conf{port} "; + } + $connection_command .= "$conf{user}\@$conf{host} $conf{server_script}"; + + if (-d ".svn") { + # Overwrite $connection_command in the case of a dev environment for now + $connection_command = "../server/scireserver.pl"; + } + + return $connection_command; +} + +sub check_job_dir { + if (! -d $conf{job_dir}) { + print "WARNING! $conf{job_dir} does not exist...creating\n"; + mkpath( $conf{job_dir}, {verbose => 1, mode => 0660}) + or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); + } +} + +sub read_config_file { my $conf_file = shift; open(FH, "< ${conf_file}") or die("Couldn't open the config file ${conf_file}: $!"); while () { @@ -143,15 +155,17 @@ sub register_client { } sub identify_client { - open FILE, "< ../etc/client_key" or die "Couldn't open client_key: $!"; - my $client_key = join("", ); - close FILE; - my $cmd = "IDENTIFY ${client_key}"; - send_command($cmd); +# open FILE, "< ../etc/client_key" or die "Couldn't open client_key: $!"; +# my $client_key = join("", ); +# close FILE; + my $client_key = "124567890"; + send_command("IDENTIFY", $client_key); my $response = get_response(); - $response =~ /^(OK|ERROR (.+))$/; + $response =~ /^(OK|ERROR)(?: (.+))?$/; if($1 eq "ERROR") { print "Could not identify to server: $2\n"; + return 0; } - print "Registered client $conf{client_id}\n" if $conf{debug}; +# print "Registered client $conf{client_id}\n" if $conf{debug}; + return 1; } -- cgit v1.2.3-65-gdbad From 81ccb2db0a61d3b6b89a9926bef210d3058977be Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 26 Dec 2007 05:18:17 +0000 Subject: copied config reading from the client added DBI. added connection code. tested, works. svn path=/branches/new-fu/; revision=264 --- server/scireserver.pl | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 2c7a718..30251e5 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -2,13 +2,25 @@ use strict; use warnings; +use DBI; +use Data::Dumper; $| = 1; -my $jobdir = "/tmp/scirejobs"; + +my $SCIRE_CONFIG_FILE = '../etc/scireserver.conf'; #will be /etc/scire.conf when released. +my %conf; + +my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; +read_config_file($conf_file); +Dumper(\%conf); +#Connect to the Database. +my $connect_string = "DBI:$conf{db_type}:$conf{db_name};host=$conf{db_host}"; +#print STDERR "Connecting to $connect_string\n"; +my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { RaiseError => 1 } ) + or die "Could not connect to database: $DBI::errstr"; while(<>) { - my $line = $_; - chomp $line; + chomp( my $line = $_); if($line =~ /^IDENTIFY (.+)$/) { my $rand_int = int(rand(3)); if($rand_int == 0) { @@ -25,3 +37,21 @@ while(<>) { print "ERROR Unknown command\n"; } } + + +sub read_config_file { + my $conf_file = shift; + open(FH, "< ${conf_file}") or die("Couldn't open the config file ${conf_file}: $!"); + while () { + chomp; + next if /^\s*(?:#|$)/; + if(/^\s*(.+?)\s*=\s*(.+?)\s*(?:#.*)?$/) { + unless(defined($conf{lc($1)})) { #Don't overwrite anything specified in cmdline + $conf{lc($1)} = $2; + } + } + } + close(FH) or die("Couldn't close the config file ${conf_file}: $!"); +# print "Conf file $conf_file read.\n"; +} + -- cgit v1.2.3-65-gdbad From 0c2526b53e0753acd006f5aad32228d1da88a2d5 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 26 Dec 2007 05:18:56 +0000 Subject: if to unless b/c of possible undef. svn path=/branches/new-fu/; revision=265 --- client/scireclient.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index fc02c54..4cd8f0e 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -162,8 +162,8 @@ sub identify_client { send_command("IDENTIFY", $client_key); my $response = get_response(); $response =~ /^(OK|ERROR)(?: (.+))?$/; - if($1 eq "ERROR") { - print "Could not identify to server: $2\n"; + unless ($1 and ($1 eq "OK")) { + print "Could not identify to server: $response\n"; return 0; } # print "Registered client $conf{client_id}\n" if $conf{debug}; -- cgit v1.2.3-65-gdbad From 2f0514049ec252157377017813d145d60948718e Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 26 Dec 2007 05:20:15 +0000 Subject: adding fake server config. svn path=/branches/new-fu/; revision=266 --- etc/scireserver.conf | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 etc/scireserver.conf diff --git a/etc/scireserver.conf b/etc/scireserver.conf new file mode 100644 index 0000000..7f77770 --- /dev/null +++ b/etc/scireserver.conf @@ -0,0 +1,8 @@ +# $Id$ +# Scire Server config file. +db_type=mysql +db_host=localhost +db_name=scire +db_user=foo +db_passwd=bar + -- cgit v1.2.3-65-gdbad From 25cb920592360a45c2b5bcc17dbf2598913c244a Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 01:11:18 +0000 Subject: adding a whole bunch of code here. lots taken from the original server code in python. svn path=/branches/new-fu/; revision=267 --- server/scireserver.pl | 148 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 130 insertions(+), 18 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 30251e5..0d3eb97 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -7,35 +7,76 @@ use Data::Dumper; $| = 1; -my $SCIRE_CONFIG_FILE = '../etc/scireserver.conf'; #will be /etc/scire.conf when released. +my $SCIRE_CONFIG_FILE = '/home/pcody/gentoo/scire-newfu/etc/scireserver.conf'; #will be /etc/scire.conf when released. my %conf; my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; read_config_file($conf_file); Dumper(\%conf); + +my $identified = 0; #Global variable to determine if already identified or not. +# Somehow this feels insecure. + #Connect to the Database. my $connect_string = "DBI:$conf{db_type}:$conf{db_name};host=$conf{db_host}"; -#print STDERR "Connecting to $connect_string\n"; -my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { RaiseError => 1 } ) - or die "Could not connect to database: $DBI::errstr"; +print STDERR "Connecting to $connect_string\n"; +#my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { RaiseError => 1 } ) +# or die "Could not connect to database: $DBI::errstr"; while(<>) { chomp( my $line = $_); - if($line =~ /^IDENTIFY (.+)$/) { - my $rand_int = int(rand(3)); - if($rand_int == 0) { - print "OK\n"; - } elsif($rand_int == 1) { - print "ERROR Unrecognized client key. Please register\n"; - } elsif($rand_int == 2) { - print "ERROR This client has not yet been authorized\n"; + if ($line =~ /^STARTRESPONSE (.+?) (.+)$/) { #If it's multi-line, parse accordingly. + print STDERR "Handling multiline response\n"; + my ($size,$md5) = ($1,$2); + my $full_cmd = ""; + while(<>) { + my $multi_line = $_; + last if $multi_line =~ /^ENDRESPONSE$/; + $full_cmd .= $multi_line; + } + #Verify size and md5. FIXME agaffney can you code this part? + #FIXME code what to do. no use yet. will be used for returning jobs. + + } else { #Handle normal single-line commands. + print STDERR "Handling single-line response\n"; + print STDERR "DEBUG: line is: $line\n"; + if($line =~ /^QUIT$/) { + print "OK\n"; + exit; + } + + if ($line =~ /^REGISTER "(.+?)" "(.+)"$/) { + my ($mac,$ip) = ($1, $2); + register_client($mac, $ip); + next; #End switch here. You can go no further. + } + + if($line =~ /^IDENTIFY (.+)$/) { + my $fingerprint = $1; + identify_client($fingerprint); + next; #End switch here. You can go no further. + } + unless($identified == 1) { + print "ERROR This client has not yet been authorized. Please identify!\n"; + next; + } + + if ($line =~ /^GET_JOBS(.*)$/) { + my @existing_jobs = split(/ /,$1) if defined($1); + get_jobs(@existing_jobs); + + } elsif ($line =~ /^GET_JOB (.+)$/) { + my $job = $1; + get_job($job); + + } elsif ($line =~ /^SET_JOB_STATUS (.+?) "(.+)"$/) { + my ($jobid,$status) = ($1, $2); + set_job_status($jobid,$status); + + } else { + print "ERROR This command $line, is unknown. Please try again.\n"; + } } - } elsif($line =~ /^QUIT$/) { - print "OK\n"; - exit; - } else { - print "ERROR Unknown command\n"; - } } @@ -55,3 +96,74 @@ sub read_config_file { # print "Conf file $conf_file read.\n"; } +#New clients must be registered so they can be given a key to use (perhaps for job file transfers?) for authentication. This must be allowed before identifying. +sub register_client { + my ($mac,$ip) = @_; + #Validate your inputs! + $mac =~ /^[a-zA-Z0-9\:]+$/ or print "ERROR invalid mac $mac!\n"; + $ip =~ /^[a-zA-Z0-9\.\:]+$/ or print "ERROR invalid ip $ip!\n"; + + my ($query, $status_id, $id); + eval { + $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; + print STDERR "DEBUG: Query is $query\n"; + $status_id = "4"; #db.conn.GetRow($query) + }; + ($@) and print "ERROR Could not get status id: $DBI::errstr"; + + eval { + $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; + print STDERR "DEBUG: Query is $query\n"; + #execute it + $query = 'SELECT id FROM `gacl_axo_seq`'; + print STDERR "DEBUG: Query is $query\n"; + $id = "56"; #execute $query + $query = 'UPDATE `gacl_axo_seq` SET id=%s'; + print STDERR "DEBUG: Query is $query\n"; + #execute with $id + $query = 'UNLOCK TABLES'; + print STDERR "DEBUG: Query is $query\n"; + }; + ($@) and print "ERROR during fetching of id sequence: $DBI::errstr"; + + eval { + $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (%s,"clients",%s,1,%s,0)'; + print STDERR "DEBUG: Query is $query\n"; + #execute with $id, $hostname, $hostname + #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. + + $query = 'INSERT INTO clients (clientid,digest,cert,hostname,mac,ip,status) VALUES (%s,%s,%s,%s,%s,%s,%s)'; + print STDERR "DEBUG: Query is $query\n"; + #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) + }; + ($@) and print "ERROR Could not insert client with $query: $DBI::errstr"; + + print "OK\n"; +} + + +#Identify the client by looking up the fingerprint in the database, and matching it up. +sub identify_client { + my $fingerprint = shift; + $fingerprint =~ s/"//g; #Clear the quotes. + $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; + #Validate your inputs! + my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=%s'; + print STDERR "Query is $query , key is $fingerprint\n"; + $identified = 1; + print "OK\n"; +} +sub get_jobs { + my (@existing_jobs) = (@_); + #Validate your inputs! + + my $query; +} +sub get_job { + my $job = shift; + #Validate your inputs! +} +sub set_job_status { + my ($jobid,$status) = @_; + #Validate your inputs! +} -- cgit v1.2.3-65-gdbad From 9007092dd569f87197f45b0588a53886391e1610 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 01:12:03 +0000 Subject: big changes. adding a whole bunch of code to flesh this out. svn path=/branches/new-fu/; revision=268 --- client/scireclient.pl | 103 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 15 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 4cd8f0e..2b961e2 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -9,6 +9,8 @@ use IPC::Open2; use Getopt::Long; use Data::Dumper; use File::Path; +#use Net::SSH::Perl::Key; + my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; @@ -30,24 +32,29 @@ sub run_main { create_connection($connection_command); #2. Register with the DB. (only it knows if you're allowed to be active) -# if(-f "../etc/client_key") { + # If we do not have a defined key file, we assume this is the first run of this client + # so we register them instead of trying to identify. + if(defined($conf{key_file}) and (-f $conf{key_file})) { if(!identify_client()) { exit(1); } -# } else { -# register_client(); -# } + } else { + register_client(); + exit(0); + } #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. + my @existing_jobs; + @existing_jobs = scan_jobs_dir(); #4. Fetch the jobs list + get_jobs(@existing_jobs); #5. ? - run_test(); + #run_test(); } sub run_test { for(('PING', 'FOO', 'QUIT')) { - send_command($_); - my $response = get_response(); + my $response = send_command($_); } } @@ -91,6 +98,10 @@ sub send_command { } print "Sending: ${tosend}\n" if $conf{debug}; print SERVER_STDIN "${tosend}\n"; + #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! + #if the server doesn't give you a newline this just hangs! + my $response = get_response(); + return $response; } sub get_response { @@ -100,6 +111,11 @@ sub get_response { return $response; } +sub parse_response { + my $resp = shift; + return "Not sure how this is gonna work yet"; +} + sub create_connection { # XXX: How do we capture this error? $pid has a valid value even if the # process fails to run, since it just returns the PID of the forked perl @@ -151,21 +167,78 @@ sub read_config_file { } sub register_client { - send_command("REGISTER 00:11:22:33:44:55 192.168.2.3 myhostname"); + my $mac = "00:11:22:33:44:55"; + my $ip = "192.168.2.3"; + my $response = send_command("REGISTER",$mac,$ip); + die "Could not register client $mac w/ ip $ip. got $response" unless ($response =~ /OK/); + print "Client registered. Status is pending.\n"; } sub identify_client { -# open FILE, "< ../etc/client_key" or die "Couldn't open client_key: $!"; -# my $client_key = join("", ); -# close FILE; - my $client_key = "124567890"; - send_command("IDENTIFY", $client_key); - my $response = get_response(); + open(FILE, $conf{key_file}) or die "Couldn't open client_key $conf{key_file}: $!"; + my $client_key = join("", ); + close(FILE); + $conf{key_type} ||= "DSA"; +# my $key_obj = Net::SSH::Perl::Key->new($conf{key_type}, $client_key); +# my $fingerprint = $key_obj->fingerprint(); + + my $fingerprint = "124567890"; + my $response = send_command("IDENTIFY", $fingerprint); $response =~ /^(OK|ERROR)(?: (.+))?$/; unless ($1 and ($1 eq "OK")) { print "Could not identify to server: $response\n"; return 0; } -# print "Registered client $conf{client_id}\n" if $conf{debug}; + print "Client identified\n" if $conf{debug}; return 1; } + +sub get_jobs { + my @existing_jobs = @_; + my $response = send_command("GET_JOBS", @existing_jobs); + $response =~ /^(OK|ERROR)(?: (.+))?$/; + unless ($1 and ($1 eq "OK")) { + print "Could not get jobs list from server: $response\n"; + return 0; + } + my $jobs = $2; + $jobs =~ s/\s//g; #Remove all whitespace + my @jobs_list = split(/,/, $jobs); + foreach my $job (@jobs_list) { + my $resp = send_command("GET_JOB",$job); + open(JOBFILE, ">$conf{job_dir}/queue/${job}.job") or do { + print "Could not open $conf{job_dir}/queue/${job}.job for writing: $!"; + next; + }; + print JOBFILE parse_response($resp); + close(JOBFILE); + print "Fetched job $job \n" if $conf{debug}; + } + #This function doesn't actually need to do anything with the list of jobs, the executor handles that part. +} + +sub scan_jobs_dir { + #Scan the dirs for job files. + my @existing_jobs = glob("$conf{job_dir}/queue/*"); + my @failed_jobs = glob("$conf{job_dir}/failed/*"); + my @done_jobs = glob("$conf{job_dir}/done/*"); + + #Report on those jobs needing reporting. + foreach my $job_file (@failed_jobs) { + $job_file =~ /(\d+)\.job/; + my $jobid = $1; + my $response = send_command("SET_JOB_STATUS $jobid 'Failed'"); + open(FILE, $job_file) or die "Couldn't open job file $job_file: $!"; + my $job_data = join("", ); + close(FILE); + + } + #may be able to use same code as above. + foreach my $job_file (@done_jobs) { + $job_file =~ /(\d+)\.job/; + my $jobid = $1; + my $response = send_command("SET_JOB_STATUS $jobid 'Done'"); + } + + return @existing_jobs; +} -- cgit v1.2.3-65-gdbad From 72b5d660f8c419b7dc00fb32256916f412243a85 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 01:15:05 +0000 Subject: config updates. svn path=/branches/new-fu/; revision=269 --- etc/scire.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/scire.conf b/etc/scire.conf index 5d01d27..87bfef3 100644 --- a/etc/scire.conf +++ b/etc/scire.conf @@ -2,4 +2,5 @@ HOST=localhost PORT=22 USER=scire SERVER_SCRIPT=../server/scireserver.pl -JOB_DIR=/tmp/scirejobs # Comments at the end of the line are a-okay +JOB_DIR=/tmp/scirejobs #comments are fine. +KEY_FILE=../etc/client.key -- cgit v1.2.3-65-gdbad From abbe117036610af5a2c158914cf764342f1ab801 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 01:43:20 +0000 Subject: move get_response() code into send_command() since nothing else uses it svn path=/branches/new-fu/; revision=270 --- client/scireclient.pl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 2b961e2..6b4ec1a 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -100,17 +100,17 @@ sub send_command { print SERVER_STDIN "${tosend}\n"; #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! #if the server doesn't give you a newline this just hangs! - my $response = get_response(); - return $response; -} - -sub get_response { - # XXX: Add some logic for multi-line responses here my $response = ; - print "Got: ${response}" if($conf{debug}); +# my $response = get_response(); return $response; } +#sub get_response { +# my $response = ; +# print "Got: ${response}" if($conf{debug}); +# return $response; +#} + sub parse_response { my $resp = shift; return "Not sure how this is gonna work yet"; -- cgit v1.2.3-65-gdbad From 7c69d7de2df4b0e74dce06f0f66372845339103d Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 01:53:28 +0000 Subject: implement parse_response() svn path=/branches/new-fu/; revision=271 --- client/scireclient.pl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 6b4ec1a..8b5ff0c 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -44,8 +44,7 @@ sub run_main { } #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. - my @existing_jobs; - @existing_jobs = scan_jobs_dir(); + my @existing_jobs = scan_jobs_dir(); #4. Fetch the jobs list get_jobs(@existing_jobs); #5. ? @@ -112,8 +111,10 @@ sub send_command { #} sub parse_response { - my $resp = shift; - return "Not sure how this is gonna work yet"; + my $response = shift; + $response =~ /^(OK|ERROR)(?: (.+))?$/; + my ($status, $message) = ($1, $2); + return ($status, $message); } sub create_connection { -- cgit v1.2.3-65-gdbad From 25bf8fee994306ca7661487297244799c3e98152 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 01:59:45 +0000 Subject: implement debug() and modify all current code to use it svn path=/branches/new-fu/; revision=272 --- client/scireclient.pl | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 8b5ff0c..c4a8efb 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -95,7 +95,7 @@ sub send_command { $tosend .= " \"${arg}\""; } } - print "Sending: ${tosend}\n" if $conf{debug}; + debug("Sending: ${tosend}\n"); print SERVER_STDIN "${tosend}\n"; #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! #if the server doesn't give you a newline this just hangs! @@ -170,9 +170,10 @@ sub read_config_file { sub register_client { my $mac = "00:11:22:33:44:55"; my $ip = "192.168.2.3"; - my $response = send_command("REGISTER",$mac,$ip); - die "Could not register client $mac w/ ip $ip. got $response" unless ($response =~ /OK/); - print "Client registered. Status is pending.\n"; +# my $response = send_command("REGISTER",$mac,$ip); + my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip)); + die "Could not register client $mac w/ ip $ip. got $response" if (! defined $status or $status ne "OK"); + debug("Client registered. Status is pending.\n"); } sub identify_client { @@ -190,7 +191,7 @@ sub identify_client { print "Could not identify to server: $response\n"; return 0; } - print "Client identified\n" if $conf{debug}; + debug("Client identified\n"); return 1; } @@ -213,7 +214,7 @@ sub get_jobs { }; print JOBFILE parse_response($resp); close(JOBFILE); - print "Fetched job $job \n" if $conf{debug}; + debug("Fetched job $job \n"); } #This function doesn't actually need to do anything with the list of jobs, the executor handles that part. } @@ -243,3 +244,10 @@ sub scan_jobs_dir { return @existing_jobs; } + +sub debug { + my $msg = shift; + if($conf{debug}) { + print STDERR $msg; + } +} -- cgit v1.2.3-65-gdbad From ac02a394dfb80ad95f93aaa5793388d5ae1c5932 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 02:03:37 +0000 Subject: modify all code using send_command() to pass output through parse_response() svn path=/branches/new-fu/; revision=273 --- client/scireclient.pl | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index c4a8efb..785cfa7 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -170,7 +170,6 @@ sub read_config_file { sub register_client { my $mac = "00:11:22:33:44:55"; my $ip = "192.168.2.3"; -# my $response = send_command("REGISTER",$mac,$ip); my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip)); die "Could not register client $mac w/ ip $ip. got $response" if (! defined $status or $status ne "OK"); debug("Client registered. Status is pending.\n"); @@ -185,9 +184,8 @@ sub identify_client { # my $fingerprint = $key_obj->fingerprint(); my $fingerprint = "124567890"; - my $response = send_command("IDENTIFY", $fingerprint); - $response =~ /^(OK|ERROR)(?: (.+))?$/; - unless ($1 and ($1 eq "OK")) { + my ($status, $message) = parse_response(send_command("IDENTIFY", $fingerprint)); + unless (defined $status && $status eq "OK") { print "Could not identify to server: $response\n"; return 0; } @@ -197,9 +195,8 @@ sub identify_client { sub get_jobs { my @existing_jobs = @_; - my $response = send_command("GET_JOBS", @existing_jobs); - $response =~ /^(OK|ERROR)(?: (.+))?$/; - unless ($1 and ($1 eq "OK")) { + my ($status, $message) = parse_response(send_command("GET_JOBS", @existing_jobs)); + unless (defined $status && $status eq "OK") { print "Could not get jobs list from server: $response\n"; return 0; } @@ -207,11 +204,12 @@ sub get_jobs { $jobs =~ s/\s//g; #Remove all whitespace my @jobs_list = split(/,/, $jobs); foreach my $job (@jobs_list) { - my $resp = send_command("GET_JOB",$job); + my ($status, $message) = parse_response(send_command("GET_JOB", $job)); open(JOBFILE, ">$conf{job_dir}/queue/${job}.job") or do { print "Could not open $conf{job_dir}/queue/${job}.job for writing: $!"; next; }; + # XXX: Modify this to fetch a file instead print JOBFILE parse_response($resp); close(JOBFILE); debug("Fetched job $job \n"); @@ -229,7 +227,7 @@ sub scan_jobs_dir { foreach my $job_file (@failed_jobs) { $job_file =~ /(\d+)\.job/; my $jobid = $1; - my $response = send_command("SET_JOB_STATUS $jobid 'Failed'"); + my ($status, $message) = parse_response(send_command("SET_JOB_STATUS $jobid 'Failed'")); open(FILE, $job_file) or die "Couldn't open job file $job_file: $!"; my $job_data = join("", ); close(FILE); @@ -239,7 +237,7 @@ sub scan_jobs_dir { foreach my $job_file (@done_jobs) { $job_file =~ /(\d+)\.job/; my $jobid = $1; - my $response = send_command("SET_JOB_STATUS $jobid 'Done'"); + my ($status, $message) = parse_response(send_command("SET_JOB_STATUS $jobid 'Done'")); } return @existing_jobs; -- cgit v1.2.3-65-gdbad From d4e036be95cbe9d3a4538440269e2ded73fbb74d Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 02:04:34 +0000 Subject: remove run_test() stuff svn path=/branches/new-fu/; revision=274 --- client/scireclient.pl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 785cfa7..d7843bc 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -48,13 +48,6 @@ sub run_main { #4. Fetch the jobs list get_jobs(@existing_jobs); #5. ? - #run_test(); -} - -sub run_test { - for(('PING', 'FOO', 'QUIT')) { - my $response = send_command($_); - } } sub parse_command_line { -- cgit v1.2.3-65-gdbad From a0fbf1bfea660b8ebdf2285ed707dbe3e5f7f3e9 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 02:19:34 +0000 Subject: relative config file location svn path=/branches/new-fu/; revision=275 --- server/scireserver.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 0d3eb97..358ba3e 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -7,7 +7,7 @@ use Data::Dumper; $| = 1; -my $SCIRE_CONFIG_FILE = '/home/pcody/gentoo/scire-newfu/etc/scireserver.conf'; #will be /etc/scire.conf when released. +my $SCIRE_CONFIG_FILE = '../etc/scireserver.conf'; #will be /etc/scire.conf when released. my %conf; my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; -- cgit v1.2.3-65-gdbad From 9f5ba833bb3718b5ed61243fbd88c081b2cc5564 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 02:23:53 +0000 Subject: add END block and clean up errors svn path=/branches/new-fu/; revision=276 --- client/scireclient.pl | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index d7843bc..59ca042 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -10,11 +10,12 @@ use Getopt::Long; use Data::Dumper; use File::Path; #use Net::SSH::Perl::Key; - +use POSIX ":sys_wait_h"; my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; my ($SERVER_STDOUT, $SERVER_STDIN); +my $connection_pid; run_main(); @@ -47,7 +48,8 @@ sub run_main { my @existing_jobs = scan_jobs_dir(); #4. Fetch the jobs list get_jobs(@existing_jobs); - #5. ? + #5. ??? + #6. Profit! } sub parse_command_line { @@ -93,7 +95,6 @@ sub send_command { #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! #if the server doesn't give you a newline this just hangs! my $response = ; -# my $response = get_response(); return $response; } @@ -118,7 +119,7 @@ sub create_connection { # to STDERR: # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 my $connection_command = shift; - my $pid = open2(*SERVER_STDOUT, *SERVER_STDIN, $connection_command); + $connection_pid = open2(*SERVER_STDOUT, *SERVER_STDIN, $connection_command); } sub build_connection_command { @@ -164,7 +165,7 @@ sub register_client { my $mac = "00:11:22:33:44:55"; my $ip = "192.168.2.3"; my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip)); - die "Could not register client $mac w/ ip $ip. got $response" if (! defined $status or $status ne "OK"); + die "Could not register client $mac w/ ip $ip. Got: $message" if (! defined $status or $status ne "OK"); debug("Client registered. Status is pending.\n"); } @@ -179,7 +180,7 @@ sub identify_client { my $fingerprint = "124567890"; my ($status, $message) = parse_response(send_command("IDENTIFY", $fingerprint)); unless (defined $status && $status eq "OK") { - print "Could not identify to server: $response\n"; + print "Could not identify to server: $message\n"; return 0; } debug("Client identified\n"); @@ -190,7 +191,7 @@ sub get_jobs { my @existing_jobs = @_; my ($status, $message) = parse_response(send_command("GET_JOBS", @existing_jobs)); unless (defined $status && $status eq "OK") { - print "Could not get jobs list from server: $response\n"; + print "Could not get jobs list from server: $message\n"; return 0; } my $jobs = $2; @@ -203,7 +204,7 @@ sub get_jobs { next; }; # XXX: Modify this to fetch a file instead - print JOBFILE parse_response($resp); +# print JOBFILE parse_response($resp); close(JOBFILE); debug("Fetched job $job \n"); } @@ -242,3 +243,9 @@ sub debug { print STDERR $msg; } } + +END { + while(waitpid($connection_pid, WNOHANG)) { + kill($connection_pid); + } +} -- cgit v1.2.3-65-gdbad From 830c2c1081d3e48aa2bee499c2f2fb0158f8a86e Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 02:35:25 +0000 Subject: remove END block as it seems unnecessary, since it looks like ssh dies by itself when the STDIN and STDOUT are closed svn path=/branches/new-fu/; revision=277 --- client/scireclient.pl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 59ca042..8c8bdff 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -10,7 +10,6 @@ use Getopt::Long; use Data::Dumper; use File::Path; #use Net::SSH::Perl::Key; -use POSIX ":sys_wait_h"; my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; @@ -243,9 +242,3 @@ sub debug { print STDERR $msg; } } - -END { - while(waitpid($connection_pid, WNOHANG)) { - kill($connection_pid); - } -} -- cgit v1.2.3-65-gdbad From 2ab61de7b4fde6f783722773c7fb3845cb51b9fa Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 02:44:17 +0000 Subject: removing hte getresponse stuff. svn path=/branches/new-fu/; revision=278 --- server/scireserver.pl | 83 ++++++++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 358ba3e..d7181bd 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -25,61 +25,48 @@ print STDERR "Connecting to $connect_string\n"; while(<>) { chomp( my $line = $_); - if ($line =~ /^STARTRESPONSE (.+?) (.+)$/) { #If it's multi-line, parse accordingly. - print STDERR "Handling multiline response\n"; - my ($size,$md5) = ($1,$2); - my $full_cmd = ""; - while(<>) { - my $multi_line = $_; - last if $multi_line =~ /^ENDRESPONSE$/; - $full_cmd .= $multi_line; - } - #Verify size and md5. FIXME agaffney can you code this part? - #FIXME code what to do. no use yet. will be used for returning jobs. - - } else { #Handle normal single-line commands. - print STDERR "Handling single-line response\n"; - print STDERR "DEBUG: line is: $line\n"; - if($line =~ /^QUIT$/) { - print "OK\n"; - exit; - } + print STDERR "Handling single-line response\n"; + print STDERR "DEBUG: line is: $line\n"; + if($line =~ /^QUIT$/) { + print "OK\n"; + exit; + } - if ($line =~ /^REGISTER "(.+?)" "(.+)"$/) { - my ($mac,$ip) = ($1, $2); - register_client($mac, $ip); - next; #End switch here. You can go no further. - } + if ($line =~ /^REGISTER "(.+?)" "(.+)"$/) { + my ($mac,$ip) = ($1, $2); + register_client($mac, $ip); + next; #End switch here. You can go no further. + } - if($line =~ /^IDENTIFY (.+)$/) { - my $fingerprint = $1; - identify_client($fingerprint); - next; #End switch here. You can go no further. - } - unless($identified == 1) { - print "ERROR This client has not yet been authorized. Please identify!\n"; - next; - } + if($line =~ /^IDENTIFY (.+)$/) { + my $fingerprint = $1; + identify_client($fingerprint); + next; #End switch here. You can go no further. + } + unless($identified == 1) { + print "ERROR This client has not yet been authorized. Please identify!\n"; + next; + } - if ($line =~ /^GET_JOBS(.*)$/) { - my @existing_jobs = split(/ /,$1) if defined($1); - get_jobs(@existing_jobs); - - } elsif ($line =~ /^GET_JOB (.+)$/) { - my $job = $1; - get_job($job); - - } elsif ($line =~ /^SET_JOB_STATUS (.+?) "(.+)"$/) { - my ($jobid,$status) = ($1, $2); - set_job_status($jobid,$status); - - } else { - print "ERROR This command $line, is unknown. Please try again.\n"; - } + if ($line =~ /^GET_JOBS(.*)$/) { + my @existing_jobs = split(/ /,$1) if defined($1); + get_jobs(@existing_jobs); + + } elsif ($line =~ /^GET_JOB (.+)$/) { + my $job = $1; + get_job($job); + + } elsif ($line =~ /^SET_JOB_STATUS (.+?) "(.+)"$/) { + my ($jobid,$status) = ($1, $2); + set_job_status($jobid,$status); + + } else { + print "ERROR This command $line, is unknown. Please try again.\n"; } } + sub read_config_file { my $conf_file = shift; open(FH, "< ${conf_file}") or die("Couldn't open the config file ${conf_file}: $!"); -- cgit v1.2.3-65-gdbad From 0412d8d45ee1a127672af6e88bd0c630849de72d Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 02:54:19 +0000 Subject: adding debug. svn path=/branches/new-fu/; revision=279 --- etc/scireserver.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/scireserver.conf b/etc/scireserver.conf index 7f77770..6755784 100644 --- a/etc/scireserver.conf +++ b/etc/scireserver.conf @@ -1,5 +1,7 @@ # $Id$ # Scire Server config file. +debug=1 +#Database stuff. db_type=mysql db_host=localhost db_name=scire -- cgit v1.2.3-65-gdbad From d25a3e29ece463e9b79a17e8d0e7cc25969aa144 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 02:54:46 +0000 Subject: adding debug() function. makes things so much cleaner. svn path=/branches/new-fu/; revision=280 --- server/scireserver.pl | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index d7181bd..6ea7543 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -17,16 +17,22 @@ Dumper(\%conf); my $identified = 0; #Global variable to determine if already identified or not. # Somehow this feels insecure. +sub debug { + my $line = shift; + if ($conf{debug}) { + print STDERR "$line\n"; + } +} + #Connect to the Database. my $connect_string = "DBI:$conf{db_type}:$conf{db_name};host=$conf{db_host}"; -print STDERR "Connecting to $connect_string\n"; +debug("Connecting to $connect_string"); #my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { RaiseError => 1 } ) # or die "Could not connect to database: $DBI::errstr"; while(<>) { chomp( my $line = $_); - print STDERR "Handling single-line response\n"; - print STDERR "DEBUG: line is: $line\n"; + debug("DEBUG: line is: $line"); if($line =~ /^QUIT$/) { print "OK\n"; exit; @@ -83,6 +89,7 @@ sub read_config_file { # print "Conf file $conf_file read.\n"; } + #New clients must be registered so they can be given a key to use (perhaps for job file transfers?) for authentication. This must be allowed before identifying. sub register_client { my ($mac,$ip) = @_; @@ -93,34 +100,34 @@ sub register_client { my ($query, $status_id, $id); eval { $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); $status_id = "4"; #db.conn.GetRow($query) }; ($@) and print "ERROR Could not get status id: $DBI::errstr"; eval { $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); #execute it $query = 'SELECT id FROM `gacl_axo_seq`'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); $id = "56"; #execute $query $query = 'UPDATE `gacl_axo_seq` SET id=%s'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); #execute with $id $query = 'UNLOCK TABLES'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); }; ($@) and print "ERROR during fetching of id sequence: $DBI::errstr"; eval { $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (%s,"clients",%s,1,%s,0)'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); #execute with $id, $hostname, $hostname #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. $query = 'INSERT INTO clients (clientid,digest,cert,hostname,mac,ip,status) VALUES (%s,%s,%s,%s,%s,%s,%s)'; - print STDERR "DEBUG: Query is $query\n"; + debug("DEBUG: Query is $query"); #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) }; ($@) and print "ERROR Could not insert client with $query: $DBI::errstr"; @@ -136,7 +143,7 @@ sub identify_client { $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; #Validate your inputs! my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=%s'; - print STDERR "Query is $query , key is $fingerprint\n"; + debug("DEBUG: Query is $query"); $identified = 1; print "OK\n"; } -- cgit v1.2.3-65-gdbad From 847a52beadb6e651e51bbf8bbe3a3746e11b776b Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 03:41:53 +0000 Subject: adding in the DBI commands, commented out, to actually do stuff instead of faking it. will continue faking it for now to ease development. svn path=/branches/new-fu/; revision=281 --- server/scireserver.pl | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 6ea7543..88f17a3 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -97,11 +97,14 @@ sub register_client { $mac =~ /^[a-zA-Z0-9\:]+$/ or print "ERROR invalid mac $mac!\n"; $ip =~ /^[a-zA-Z0-9\.\:]+$/ or print "ERROR invalid ip $ip!\n"; - my ($query, $status_id, $id); + my ($query, $status_id, $id, $sth); eval { $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; debug("DEBUG: Query is $query"); $status_id = "4"; #db.conn.GetRow($query) + #$sth = $dbh->prepare($query); + #my @result = $sth->fetchrow_array(); + #$status_id = $result[0]; }; ($@) and print "ERROR Could not get status id: $DBI::errstr"; @@ -109,26 +112,38 @@ sub register_client { $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; debug("DEBUG: Query is $query"); #execute it + #$dbh->do($query); $query = 'SELECT id FROM `gacl_axo_seq`'; debug("DEBUG: Query is $query"); $id = "56"; #execute $query - $query = 'UPDATE `gacl_axo_seq` SET id=%s'; + #$sth = $dbh->prepare($query); + #my @result = $sth->fetchrow_array(); + #$id = $result[0]; + + $query = 'UPDATE `gacl_axo_seq` SET id=?'; debug("DEBUG: Query is $query"); #execute with $id + #$sth = $dbh->prepare($query); + #$sth->execute($id); $query = 'UNLOCK TABLES'; debug("DEBUG: Query is $query"); + #$dbh->do($query); }; ($@) and print "ERROR during fetching of id sequence: $DBI::errstr"; eval { - $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (%s,"clients",%s,1,%s,0)'; + $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (?,"clients",?,1,?,0)'; debug("DEBUG: Query is $query"); + #$sth = $dbh->prepare($query); + #$sth->execute($id, $hostname, $hostname); #execute with $id, $hostname, $hostname #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. - $query = 'INSERT INTO clients (clientid,digest,cert,hostname,mac,ip,status) VALUES (%s,%s,%s,%s,%s,%s,%s)'; + $query = 'INSERT INTO clients (clientid,digest,cert,hostname,mac,ip,status) VALUES (?,?,?,?,?,?,?)'; debug("DEBUG: Query is $query"); #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) + #$sth = $dbh->prepare($query); + #$sth->execute($id,$digest,$hostname,$mac,$ip,$status_id); }; ($@) and print "ERROR Could not insert client with $query: $DBI::errstr"; @@ -142,8 +157,10 @@ sub identify_client { $fingerprint =~ s/"//g; #Clear the quotes. $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; #Validate your inputs! - my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=%s'; + my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; debug("DEBUG: Query is $query"); + #$sth = $dbh->prepare($query); + #$sth->execute($fingerprint); $identified = 1; print "OK\n"; } -- cgit v1.2.3-65-gdbad From da01a76bee1e6735c036baed76681dcfe9c3ca43 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 30 Dec 2007 06:19:03 +0000 Subject: adding query for get_jobs from old server. svn path=/branches/new-fu/; revision=282 --- server/scireserver.pl | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 88f17a3..6dd9a8c 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -106,7 +106,7 @@ sub register_client { #my @result = $sth->fetchrow_array(); #$status_id = $result[0]; }; - ($@) and print "ERROR Could not get status id: $DBI::errstr"; + ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; eval { $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; @@ -129,7 +129,7 @@ sub register_client { debug("DEBUG: Query is $query"); #$dbh->do($query); }; - ($@) and print "ERROR during fetching of id sequence: $DBI::errstr"; + ($@) and print "ERROR during fetching of id sequence: $DBI::errstr\n"; eval { $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (?,"clients",?,1,?,0)'; @@ -145,7 +145,7 @@ sub register_client { #$sth = $dbh->prepare($query); #$sth->execute($id,$digest,$hostname,$mac,$ip,$status_id); }; - ($@) and print "ERROR Could not insert client with $query: $DBI::errstr"; + ($@) and print "ERROR Could not insert client with $query: $DBI::errstr\n"; print "OK\n"; } @@ -154,9 +154,10 @@ sub register_client { #Identify the client by looking up the fingerprint in the database, and matching it up. sub identify_client { my $fingerprint = shift; + #Validate your inputs! $fingerprint =~ s/"//g; #Clear the quotes. $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; - #Validate your inputs! + my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; debug("DEBUG: Query is $query"); #$sth = $dbh->prepare($query); @@ -167,8 +168,23 @@ sub identify_client { sub get_jobs { my (@existing_jobs) = (@_); #Validate your inputs! - - my $query; + foreach my $jobid (@existing_jobs) { + unless($jobid =~ /^\d$/) { + print "ERROR Invalid jobid given as input $jobid\n"; + return undef; + } + } + my $query = <<'EndOfQuery' + SELECT jobs.jobid, jobs.priority, job_conditions.job_dependency, job_conditions.deploy_time, job_conditions.expiration_time, job_history.statusid + FROM jobs NATURAL JOIN jobs_clients NATURAL JOIN job_conditions NATURAL JOIN job_history + WHERE jobs_clients.clientid = %s + AND jobs.jobid = jobs_clients.jobid + AND (job_conditions.deploy_time < now()) + AND (job_conditions.expiration_time > now()) + AND job_history.statusid = '%s' + ORDER BY jobs.priority,jobs.created +EndOfQuery + } sub get_job { my $job = shift; -- cgit v1.2.3-65-gdbad From 818aeed693588286d34eed62bebdfa8833f36c86 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 17:00:57 +0000 Subject: add parse_command() and modify existing code to use it svn path=/branches/new-fu/; revision=283 --- server/scireserver.pl | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 6dd9a8c..2d5f11d 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -31,21 +31,23 @@ debug("Connecting to $connect_string"); # or die "Could not connect to database: $DBI::errstr"; while(<>) { - chomp( my $line = $_); - debug("DEBUG: line is: $line"); - if($line =~ /^QUIT$/) { + my ($command, @args) = parse_command($_); +# chomp( my $line = $_); +# debug("DEBUG: line is: $line"); + + if($command eq "QUIT") { print "OK\n"; exit; } - if ($line =~ /^REGISTER "(.+?)" "(.+)"$/) { - my ($mac,$ip) = ($1, $2); + if($command eq "REGISTER") { + my ($mac,$ip) = @args; register_client($mac, $ip); next; #End switch here. You can go no further. } - if($line =~ /^IDENTIFY (.+)$/) { - my $fingerprint = $1; + if($command eq "IDENTIFY") { + my $fingerprint = $args[0]; identify_client($fingerprint); next; #End switch here. You can go no further. } @@ -54,20 +56,20 @@ while(<>) { next; } - if ($line =~ /^GET_JOBS(.*)$/) { - my @existing_jobs = split(/ /,$1) if defined($1); + if ($command eq "GET_JOBS") { + my @existing_jobs = @args; get_jobs(@existing_jobs); - - } elsif ($line =~ /^GET_JOB (.+)$/) { - my $job = $1; + + } elsif ($command eq "GET_JOB") { + my $job = $args[0]; get_job($job); - } elsif ($line =~ /^SET_JOB_STATUS (.+?) "(.+)"$/) { - my ($jobid,$status) = ($1, $2); + } elsif ($command eq "SET_JOB_STATUS") { + my ($jobid,$status) = @args; set_job_status($jobid,$status); } else { - print "ERROR This command $line, is unknown. Please try again.\n"; + print "ERROR The command $command is unknown. Please try again.\n"; } } @@ -194,3 +196,14 @@ sub set_job_status { my ($jobid,$status) = @_; #Validate your inputs! } + +sub parse_command { + my $line = shift; + chomp $line; + my @parts = split / (?!(?:[^" ]|[^"] [^"])+")/, $line; + for(0..$#parts) { + $parts[$_] =~ s/(^"|"$)//g; + $parts[$_] =~ s/\\"/"/g; + } + return @parts; +} -- cgit v1.2.3-65-gdbad From 86c88b4289b19479a2f1d0ca5e554ea06e9aa9d0 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 21:35:51 +0000 Subject: log server debug output to a file svn path=/branches/new-fu/; revision=284 --- etc/scireserver.conf | 1 + server/scireserver.pl | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/etc/scireserver.conf b/etc/scireserver.conf index 6755784..298de1e 100644 --- a/etc/scireserver.conf +++ b/etc/scireserver.conf @@ -1,6 +1,7 @@ # $Id$ # Scire Server config file. debug=1 +logfile=/tmp/scireserver.log #Database stuff. db_type=mysql db_host=localhost diff --git a/server/scireserver.pl b/server/scireserver.pl index 2d5f11d..3c39b8c 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -9,6 +9,7 @@ $| = 1; my $SCIRE_CONFIG_FILE = '../etc/scireserver.conf'; #will be /etc/scire.conf when released. my %conf; +my $LOGFILE; my $conf_file = (defined($conf{config})) ? $conf{config} : $SCIRE_CONFIG_FILE; read_config_file($conf_file); @@ -17,10 +18,18 @@ Dumper(\%conf); my $identified = 0; #Global variable to determine if already identified or not. # Somehow this feels insecure. +sub logger { + my $line = shift; + if(!defined $LOGFILE) { + open(*LOGFILE, ">>$conf{logfile}") or die "Cannot open logfile $conf{logfile}"; + } + print LOGFILE localtime() . " " . $line . "\n"; +} + sub debug { my $line = shift; if ($conf{debug}) { - print STDERR "$line\n"; + logger("DEBUG: ${line}"); } } @@ -102,7 +111,7 @@ sub register_client { my ($query, $status_id, $id, $sth); eval { $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); $status_id = "4"; #db.conn.GetRow($query) #$sth = $dbh->prepare($query); #my @result = $sth->fetchrow_array(); @@ -112,37 +121,37 @@ sub register_client { eval { $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); #execute it #$dbh->do($query); $query = 'SELECT id FROM `gacl_axo_seq`'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); $id = "56"; #execute $query #$sth = $dbh->prepare($query); #my @result = $sth->fetchrow_array(); #$id = $result[0]; $query = 'UPDATE `gacl_axo_seq` SET id=?'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); #execute with $id #$sth = $dbh->prepare($query); #$sth->execute($id); $query = 'UNLOCK TABLES'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); #$dbh->do($query); }; ($@) and print "ERROR during fetching of id sequence: $DBI::errstr\n"; eval { $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (?,"clients",?,1,?,0)'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); #$sth = $dbh->prepare($query); #$sth->execute($id, $hostname, $hostname); #execute with $id, $hostname, $hostname #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. $query = 'INSERT INTO clients (clientid,digest,cert,hostname,mac,ip,status) VALUES (?,?,?,?,?,?,?)'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) #$sth = $dbh->prepare($query); #$sth->execute($id,$digest,$hostname,$mac,$ip,$status_id); @@ -161,7 +170,7 @@ sub identify_client { $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; - debug("DEBUG: Query is $query"); + debug("Query is $query"); #$sth = $dbh->prepare($query); #$sth->execute($fingerprint); $identified = 1; -- cgit v1.2.3-65-gdbad From ebb4b2074333148136b21d46c27b26d5b4c883c5 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 30 Dec 2007 21:39:31 +0000 Subject: debug statement svn path=/branches/new-fu/; revision=285 --- server/scireserver.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 3c39b8c..4b80497 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -97,7 +97,7 @@ sub read_config_file { } } close(FH) or die("Couldn't close the config file ${conf_file}: $!"); -# print "Conf file $conf_file read.\n"; + debug("Conf file $conf_file read."); } -- cgit v1.2.3-65-gdbad From 3c98f6fb6b3e21c8e9bc645d82cf32f3655d3938 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 01:28:36 +0000 Subject: adding a check for existence of logfile in the config if not present, falls back to STDERR. helps for the time being. svn path=/branches/new-fu/; revision=286 --- etc/scireserver.conf | 2 +- server/scireserver.pl | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/scireserver.conf b/etc/scireserver.conf index 298de1e..3f60dbc 100644 --- a/etc/scireserver.conf +++ b/etc/scireserver.conf @@ -1,7 +1,7 @@ # $Id$ # Scire Server config file. debug=1 -logfile=/tmp/scireserver.log +#logfile=/tmp/scireserver.log #Database stuff. db_type=mysql db_host=localhost diff --git a/server/scireserver.pl b/server/scireserver.pl index 4b80497..ef3933e 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -29,7 +29,11 @@ sub logger { sub debug { my $line = shift; if ($conf{debug}) { - logger("DEBUG: ${line}"); + if (defined($conf{logfile})) { + logger("DEBUG: ${line}"); + } else { + print STDERR "DEBUG: ${line}\n"; + } } } -- cgit v1.2.3-65-gdbad From 2390fb8ef38add48854d201cc21c0c9c9a8aad81 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 06:51:28 +0000 Subject: added global for clientid, added to identify query. toss existing jobs concept for get_jobs. fleshed out get_jobs and get_job a bit. fixed some rough DBI. svn path=/branches/new-fu/; revision=287 --- server/scireserver.pl | 63 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index ef3933e..dfff449 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -16,6 +16,7 @@ read_config_file($conf_file); Dumper(\%conf); my $identified = 0; #Global variable to determine if already identified or not. +my $client_id = 0; #Clobal variable for the client id. # Somehow this feels insecure. sub logger { @@ -70,8 +71,7 @@ while(<>) { } if ($command eq "GET_JOBS") { - my @existing_jobs = @args; - get_jobs(@existing_jobs); + get_jobs(); } elsif ($command eq "GET_JOB") { my $job = $args[0]; @@ -117,9 +117,8 @@ sub register_client { $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; debug("Query is $query"); $status_id = "4"; #db.conn.GetRow($query) - #$sth = $dbh->prepare($query); - #my @result = $sth->fetchrow_array(); - #$status_id = $result[0]; + #$sth = $dbh->prepare($query); + #$status_id = $sth->fetchrow_hashref->{'statusid'}; }; ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; @@ -132,8 +131,7 @@ sub register_client { debug("Query is $query"); $id = "56"; #execute $query #$sth = $dbh->prepare($query); - #my @result = $sth->fetchrow_array(); - #$id = $result[0]; + #$id = $sth->fetchrow_hashref->{'id'}; $query = 'UPDATE `gacl_axo_seq` SET id=?'; debug("Query is $query"); @@ -173,37 +171,56 @@ sub identify_client { $fingerprint =~ s/"//g; #Clear the quotes. $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; - my $query = 'SELECT client_status.statusname FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; + my $query = 'SELECT client_status.statusname, clients.clientid FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; debug("Query is $query"); #$sth = $dbh->prepare($query); - #$sth->execute($fingerprint); + #$sth->execute($digest); + #my $status_name = $sth->fetchrow_hashref->{'client_status.statusname'}; + #$client_id = $sth->fetchrow_hashref->{'clients.clientid'}; $identified = 1; print "OK\n"; + } + + sub get_jobs { - my (@existing_jobs) = (@_); - #Validate your inputs! - foreach my $jobid (@existing_jobs) { - unless($jobid =~ /^\d$/) { - print "ERROR Invalid jobid given as input $jobid\n"; - return undef; - } - } - my $query = <<'EndOfQuery' - SELECT jobs.jobid, jobs.priority, job_conditions.job_dependency, job_conditions.deploy_time, job_conditions.expiration_time, job_history.statusid + my $query = <<' EndOfQuery' + SELECT jobs.jobid FROM jobs NATURAL JOIN jobs_clients NATURAL JOIN job_conditions NATURAL JOIN job_history - WHERE jobs_clients.clientid = %s + WHERE jobs_clients.clientid = ? AND jobs.jobid = jobs_clients.jobid AND (job_conditions.deploy_time < now()) AND (job_conditions.expiration_time > now()) - AND job_history.statusid = '%s' + AND job_history.statusid = '?' ORDER BY jobs.priority,jobs.created -EndOfQuery + EndOfQuery + + debug("Query is $query"); + #$sth = $dbh->prepare($query); + #$sth->execute($client_id,$status_id); + #my $jobs_ref = $sth->fetchall_arrayref(); + #return $jobs_ref; } sub get_job { - my $job = shift; + my $jobid = shift; #Validate your inputs! + my $query = 'SELECT * FROM jobs LEFT JOIN job_conditions on (jobs.jobid) WHERE jobs.jobid = ?'; + debug("Query is $query"); + #my $sth = $dbh->prepare($query); + #$sth->execute($jobid); + #my $job = $sth->fetchrow_hashref(); + #my $scriptid = $job{'script'}; + + $query = 'SELECT * FROM scripts WHERE scriptid=?'; + debug("Query is $query"); + #$sth = $dbh->prepare($query); + #$sth->execute($scriptid); + #$job{'script'} = $sth->fetchrow_hashref(); + + #Write the job w/ all data to a jobfile with the following path /JOBDIR/CLIENT_ID/queue/JOBID.job + my $filename = "$conf{job_dir}/$client_id/queue/$jobid.job"; + return "OK $filename\n"; } sub set_job_status { my ($jobid,$status) = @_; -- cgit v1.2.3-65-gdbad From 3fa9eeff71daa927adf18b25df2b94f228b38bd1 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 06:52:05 +0000 Subject: add jobdir for server. svn path=/branches/new-fu/; revision=288 --- etc/scireserver.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/scireserver.conf b/etc/scireserver.conf index 3f60dbc..33f256a 100644 --- a/etc/scireserver.conf +++ b/etc/scireserver.conf @@ -1,6 +1,7 @@ # $Id$ # Scire Server config file. debug=1 +job_dir=/tmp/scirejobs #logfile=/tmp/scireserver.log #Database stuff. db_type=mysql -- cgit v1.2.3-65-gdbad From 99df2316bf69dfd32239224a1474f2edc245afee Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 16:38:42 +0000 Subject: add interface config option svn path=/branches/new-fu/; revision=289 --- etc/scire.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/scire.conf b/etc/scire.conf index 87bfef3..27afe64 100644 --- a/etc/scire.conf +++ b/etc/scire.conf @@ -4,3 +4,4 @@ USER=scire SERVER_SCRIPT=../server/scireserver.pl JOB_DIR=/tmp/scirejobs #comments are fine. KEY_FILE=../etc/client.key +interface=eth0 # This is the default -- cgit v1.2.3-65-gdbad From 00350150fb0e3ebde525caf270d21bb003261eef Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 16:46:19 +0000 Subject: add get_interface_info() to get MAC/IP for register_client() svn path=/branches/new-fu/; revision=290 --- client/scireclient.pl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 8c8bdff..6485bac 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -161,8 +161,9 @@ sub read_config_file { } sub register_client { - my $mac = "00:11:22:33:44:55"; - my $ip = "192.168.2.3"; +# my $mac = "00:11:22:33:44:55"; +# my $ip = "192.168.2.3"; + my ($mac, $ip) = get_interface_info(defined $conf{interface} && $conf{interface} ? $conf{interface} : "eth0"); my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip)); die "Could not register client $mac w/ ip $ip. Got: $message" if (! defined $status or $status ne "OK"); debug("Client registered. Status is pending.\n"); @@ -242,3 +243,12 @@ sub debug { print STDERR $msg; } } + +sub get_interface_info { + my $interface = shift; + + my $info = `/sbin/ifconfig ${interface}`; + $info =~ /^.+HWaddr ([a-zA-Z0-9:]+).+inet addr:([0-9.]+).+$/s; + my ($mac, $ip) = ($1, $2); + return ($mac, $ip); +} -- cgit v1.2.3-65-gdbad From 745af6195841895c1e4adf192eaae74d259084dc Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 16:53:36 +0000 Subject: drop tab in heredoc terminator svn path=/branches/new-fu/; revision=291 --- server/scireserver.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index dfff449..a71768f 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -184,7 +184,7 @@ sub identify_client { sub get_jobs { - my $query = <<' EndOfQuery' + my $query = <<'EndOfQuery'; SELECT jobs.jobid FROM jobs NATURAL JOIN jobs_clients NATURAL JOIN job_conditions NATURAL JOIN job_history WHERE jobs_clients.clientid = ? @@ -193,7 +193,7 @@ sub get_jobs { AND (job_conditions.expiration_time > now()) AND job_history.statusid = '?' ORDER BY jobs.priority,jobs.created - EndOfQuery +EndOfQuery debug("Query is $query"); #$sth = $dbh->prepare($query); -- cgit v1.2.3-65-gdbad From 7f888e372b5a2ec9c5e8a4d45397985b4f9049a4 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 17:58:41 +0000 Subject: cleaned up the debugging. took out existing jobs lines. svn path=/branches/new-fu/; revision=292 --- client/scireclient.pl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 6485bac..bfab4a5 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -46,7 +46,7 @@ sub run_main { #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. my @existing_jobs = scan_jobs_dir(); #4. Fetch the jobs list - get_jobs(@existing_jobs); + get_jobs(); #5. ??? #6. Profit! } @@ -89,7 +89,7 @@ sub send_command { $tosend .= " \"${arg}\""; } } - debug("Sending: ${tosend}\n"); + debug("Sending: ${tosend}"); print SERVER_STDIN "${tosend}\n"; #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! #if the server doesn't give you a newline this just hangs! @@ -166,7 +166,7 @@ sub register_client { my ($mac, $ip) = get_interface_info(defined $conf{interface} && $conf{interface} ? $conf{interface} : "eth0"); my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip)); die "Could not register client $mac w/ ip $ip. Got: $message" if (! defined $status or $status ne "OK"); - debug("Client registered. Status is pending.\n"); + debug("Client registered. Status is pending."); } sub identify_client { @@ -183,18 +183,16 @@ sub identify_client { print "Could not identify to server: $message\n"; return 0; } - debug("Client identified\n"); + debug("Client identified"); return 1; } sub get_jobs { - my @existing_jobs = @_; - my ($status, $message) = parse_response(send_command("GET_JOBS", @existing_jobs)); + my ($status, $jobs) = parse_response(send_command("GET_JOBS")); unless (defined $status && $status eq "OK") { print "Could not get jobs list from server: $message\n"; return 0; } - my $jobs = $2; $jobs =~ s/\s//g; #Remove all whitespace my @jobs_list = split(/,/, $jobs); foreach my $job (@jobs_list) { @@ -206,7 +204,7 @@ sub get_jobs { # XXX: Modify this to fetch a file instead # print JOBFILE parse_response($resp); close(JOBFILE); - debug("Fetched job $job \n"); + debug("Fetched job $job "); } #This function doesn't actually need to do anything with the list of jobs, the executor handles that part. } @@ -240,7 +238,7 @@ sub scan_jobs_dir { sub debug { my $msg = shift; if($conf{debug}) { - print STDERR $msg; + print STDERR $msg."\n"; } } -- cgit v1.2.3-65-gdbad From dd194e84b98bab1661f0f8ff6a3912f9b6c54aab Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 17:59:56 +0000 Subject: add the debug line to the config. svn path=/branches/new-fu/; revision=293 --- etc/scire.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/scire.conf b/etc/scire.conf index 27afe64..243e5ca 100644 --- a/etc/scire.conf +++ b/etc/scire.conf @@ -5,3 +5,4 @@ SERVER_SCRIPT=../server/scireserver.pl JOB_DIR=/tmp/scirejobs #comments are fine. KEY_FILE=../etc/client.key interface=eth0 # This is the default +debug=1 -- cgit v1.2.3-65-gdbad From 9ea19bf11220a4b6132a3937e2fbadc819746efb Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 21:38:25 +0000 Subject: add Id tag svn path=/branches/new-fu/; revision=294 --- server/scireserver.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/scireserver.pl b/server/scireserver.pl index a71768f..2e923e8 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -1,5 +1,7 @@ #!/usr/bin/perl +$ Id: $ + use strict; use warnings; use DBI; -- cgit v1.2.3-65-gdbad From 034152df7aca3f36e4ccb9c02646b6e7e29e6e23 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 21:38:53 +0000 Subject: add Id tag properly svn path=/branches/new-fu/; revision=295 --- server/scireserver.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 2e923e8..93c5f8e 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -1,6 +1,6 @@ #!/usr/bin/perl -$ Id: $ +# $Id$ use strict; use warnings; -- cgit v1.2.3-65-gdbad From 4f3643fc8469bf52bdeebec51a48b1e33eaf56c8 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 22:37:16 +0000 Subject: remove dead code add a few extra options to the ssh invocation svn path=/branches/new-fu/; revision=298 --- client/scireclient.pl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index bfab4a5..bab1ced 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -9,7 +9,6 @@ use IPC::Open2; use Getopt::Long; use Data::Dumper; use File::Path; -#use Net::SSH::Perl::Key; my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; @@ -97,12 +96,6 @@ sub send_command { return $response; } -#sub get_response { -# my $response = ; -# print "Got: ${response}" if($conf{debug}); -# return $response; -#} - sub parse_response { my $response = shift; $response =~ /^(OK|ERROR)(?: (.+))?$/; @@ -124,6 +117,9 @@ sub create_connection { sub build_connection_command { # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" my $connection_command = "ssh "; + $connection_command .= "-o BatchMode yes "; + $connection_command .= "-o SendEnv 'SCIRE_*' "; + $connection_command .= "-o ServerAliveInterval 15 -o ServerAliveCountMax 4 "; if(defined($conf{port})) { $connection_command .= "-o Port=$conf{port} "; } -- cgit v1.2.3-65-gdbad From 64bbe29ab8cc4e6dbd9e950e2666d27c707310ea Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 23:27:40 +0000 Subject: fixing up registering and identifying code. svn path=/branches/new-fu/; revision=299 --- client/scireclient.pl | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index bab1ced..490a85b 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -9,6 +9,7 @@ use IPC::Open2; use Getopt::Long; use Data::Dumper; use File::Path; +use Sys::Hostname; my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. my %conf; @@ -160,21 +161,22 @@ sub register_client { # my $mac = "00:11:22:33:44:55"; # my $ip = "192.168.2.3"; my ($mac, $ip) = get_interface_info(defined $conf{interface} && $conf{interface} ? $conf{interface} : "eth0"); - my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip)); - die "Could not register client $mac w/ ip $ip. Got: $message" if (! defined $status or $status ne "OK"); - debug("Client registered. Status is pending."); + my $hostname = hostname(); + my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip, $hostname)); + die "Could not register client $mac w/ ip $ip and hostname $hostname. Got: $message" if (! defined $status or $status ne "OK"); + debug("Client registered. Status is pending. digest is $message"); + open(FILE, ">$conf{key_file}") or die("Couldn't open key file $conf{key_file} for writing: $!"); + print FILE "$message\n"; + close(FILE); } sub identify_client { - open(FILE, $conf{key_file}) or die "Couldn't open client_key $conf{key_file}: $!"; - my $client_key = join("", ); + open(FILE, $conf{key_file}) or die("Couldn't open client_key $conf{key_file}: $!"); + my $digest = join("", ); close(FILE); - $conf{key_type} ||= "DSA"; -# my $key_obj = Net::SSH::Perl::Key->new($conf{key_type}, $client_key); -# my $fingerprint = $key_obj->fingerprint(); - my $fingerprint = "124567890"; - my ($status, $message) = parse_response(send_command("IDENTIFY", $fingerprint)); + #my $digest = "124567890"; + my ($status, $message) = parse_response(send_command("IDENTIFY", $digest)); unless (defined $status && $status eq "OK") { print "Could not identify to server: $message\n"; return 0; @@ -186,7 +188,7 @@ sub identify_client { sub get_jobs { my ($status, $jobs) = parse_response(send_command("GET_JOBS")); unless (defined $status && $status eq "OK") { - print "Could not get jobs list from server: $message\n"; + print "Could not get jobs list from server: $jobs\n"; return 0; } $jobs =~ s/\s//g; #Remove all whitespace -- cgit v1.2.3-65-gdbad From 4735f0ad8fa83690059bf7148ce6c631ac957e0b Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 31 Dec 2007 23:31:39 +0000 Subject: switching on the database lines. major fixes to register and identify code. uses a md5sum of the time+mac+ip+hostname svn path=/branches/new-fu/; revision=300 --- server/scireserver.pl | 80 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 93c5f8e..6e0d593 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -6,6 +6,7 @@ use strict; use warnings; use DBI; use Data::Dumper; +use Digest::MD5 qw(md5 md5_hex ); $| = 1; @@ -43,8 +44,8 @@ sub debug { #Connect to the Database. my $connect_string = "DBI:$conf{db_type}:$conf{db_name};host=$conf{db_host}"; debug("Connecting to $connect_string"); -#my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { RaiseError => 1 } ) -# or die "Could not connect to database: $DBI::errstr"; +my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { RaiseError => 1 } ) + or die "Could not connect to database: $DBI::errstr"; while(<>) { my ($command, @args) = parse_command($_); @@ -57,8 +58,8 @@ while(<>) { } if($command eq "REGISTER") { - my ($mac,$ip) = @args; - register_client($mac, $ip); + my ($mac,$ip,$hostname) = @args; + register_client($mac, $ip, $hostname); next; #End switch here. You can go no further. } @@ -109,18 +110,23 @@ sub read_config_file { #New clients must be registered so they can be given a key to use (perhaps for job file transfers?) for authentication. This must be allowed before identifying. sub register_client { - my ($mac,$ip) = @_; + my ($mac,$ip, $hostname) = @_; #Validate your inputs! $mac =~ /^[a-zA-Z0-9\:]+$/ or print "ERROR invalid mac $mac!\n"; $ip =~ /^[a-zA-Z0-9\.\:]+$/ or print "ERROR invalid ip $ip!\n"; my ($query, $status_id, $id, $sth); + + #Generate the digest + my $digest = md5_hex(time()."${mac}${ip}${hostname}"); + eval { $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; debug("Query is $query"); - $status_id = "4"; #db.conn.GetRow($query) - #$sth = $dbh->prepare($query); - #$status_id = $sth->fetchrow_hashref->{'statusid'}; +# $status_id = "4"; #db.conn.GetRow($query) + $sth = $dbh->prepare($query); + $sth->execute(); + $status_id = $sth->fetchrow_hashref->{'statusid'}; }; ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; @@ -128,59 +134,67 @@ sub register_client { $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; debug("Query is $query"); #execute it - #$dbh->do($query); + $dbh->do($query); $query = 'SELECT id FROM `gacl_axo_seq`'; debug("Query is $query"); - $id = "56"; #execute $query - #$sth = $dbh->prepare($query); - #$id = $sth->fetchrow_hashref->{'id'}; - + #$id = "56"; #execute $query + $sth = $dbh->prepare($query); + $sth->execute(); + $id = $sth->fetchrow_hashref->{'id'}; + $id += 1; $query = 'UPDATE `gacl_axo_seq` SET id=?'; debug("Query is $query"); #execute with $id - #$sth = $dbh->prepare($query); - #$sth->execute($id); + $sth = $dbh->prepare($query); + $sth->execute($id); $query = 'UNLOCK TABLES'; debug("Query is $query"); - #$dbh->do($query); + $dbh->do($query); }; ($@) and print "ERROR during fetching of id sequence: $DBI::errstr\n"; - + eval { - $query = 'INSERT INFO `gacl_axo` (id,section_value,value,order_value,name,hidden VALUES (?,"clients",?,1,?,0)'; + $query = 'INSERT INTO `gacl_axo` (id,section_value,value,order_value,name,hidden) VALUES (?,"clients",?,"1",?,"0")'; debug("Query is $query"); - #$sth = $dbh->prepare($query); - #$sth->execute($id, $hostname, $hostname); + $sth = $dbh->prepare($query); + $sth->execute($id, $hostname, $hostname); #execute with $id, $hostname, $hostname #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. - $query = 'INSERT INTO clients (clientid,digest,cert,hostname,mac,ip,status) VALUES (?,?,?,?,?,?,?)'; + $query = 'INSERT INTO clients (clientid,digest,hostname,mac,ip,status) VALUES (?,?,?,?,?,?)'; debug("Query is $query"); #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) - #$sth = $dbh->prepare($query); - #$sth->execute($id,$digest,$hostname,$mac,$ip,$status_id); + $sth = $dbh->prepare($query); + $sth->execute($id,$digest,$hostname,$mac,$ip,$status_id); }; ($@) and print "ERROR Could not insert client with $query: $DBI::errstr\n"; + #FIXME look for "duplicate key" and if found fail and notify admin. - print "OK\n"; + print "OK $digest\n"; } #Identify the client by looking up the fingerprint in the database, and matching it up. sub identify_client { - my $fingerprint = shift; + my $digest = shift; #Validate your inputs! - $fingerprint =~ s/"//g; #Clear the quotes. - $fingerprint =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid fingerprint!\n"; + $digest =~ s/"//g; #Clear the quotes. + $digest =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid digest!\n"; my $query = 'SELECT client_status.statusname, clients.clientid FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; debug("Query is $query"); - #$sth = $dbh->prepare($query); - #$sth->execute($digest); - #my $status_name = $sth->fetchrow_hashref->{'client_status.statusname'}; - #$client_id = $sth->fetchrow_hashref->{'clients.clientid'}; - $identified = 1; - print "OK\n"; + my $sth = $dbh->prepare($query); + $sth->execute($digest); + my $hashref = $sth->fetchrow_hashref(); + debug(Dumper($hashref)); + my $status_name = $hashref->{'statusname'}; + $client_id = $hashref->{'clientid'}; + if ($client_id > 0) { #and ($status_name eq 'Active') { + $identified = 1; + print "OK\n"; + } else { + print "ERROR Client could not be identified. Status was $status_name\n"; + } } -- cgit v1.2.3-65-gdbad From 7105b8bb9478aa4538a2346162a841ceb76059ef Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 31 Dec 2007 23:57:35 +0000 Subject: switch config location to /etc/scire/ add config defaults to scireclient svn path=/branches/new-fu/; revision=301 --- client/scireclient.pl | 12 +++++++++++- server/scireserver.pl | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 490a85b..b93c79a 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -11,7 +11,8 @@ use Data::Dumper; use File::Path; use Sys::Hostname; -my $SCIRE_CONFIG_FILE = '../etc/scire.conf'; #will be /etc/scire.conf when released. +my $ETC_DIR = "/etc/scire"; +my $SCIRE_CONFIG_FILE = "${ETC_DIR}/scire.conf"; my %conf; my ($SERVER_STDOUT, $SERVER_STDIN); my $connection_pid; @@ -144,6 +145,10 @@ sub check_job_dir { sub read_config_file { my $conf_file = shift; + my %config_defaults = ( + "key_file" => "${ETC_DIR}/client_key", + "debug" => 0, + ); open(FH, "< ${conf_file}") or die("Couldn't open the config file ${conf_file}: $!"); while () { chomp; @@ -155,6 +160,11 @@ sub read_config_file { } } close(FH) or die("Couldn't close the config file ${conf_file}: $!"); + for(keys %config_defaults) { + if(!defined $conf{$_}) { + $conf{$_} = $config_defaults{$_}; + } + } } sub register_client { diff --git a/server/scireserver.pl b/server/scireserver.pl index 6e0d593..4de6362 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -10,7 +10,8 @@ use Digest::MD5 qw(md5 md5_hex ); $| = 1; -my $SCIRE_CONFIG_FILE = '../etc/scireserver.conf'; #will be /etc/scire.conf when released. +my $ETC_DIR = "/etc/scire"; +my $SCIRE_CONFIG_FILE = "${ETC_DIR}/scireserver.conf"; my %conf; my $LOGFILE; -- cgit v1.2.3-65-gdbad From b0df877e023bb73236543528914b84112b98b209 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Tue, 1 Jan 2008 01:20:06 +0000 Subject: chomp the line read from client_key svn path=/branches/new-fu/; revision=303 --- client/scireclient.pl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index b93c79a..1da3dcd 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -182,10 +182,9 @@ sub register_client { sub identify_client { open(FILE, $conf{key_file}) or die("Couldn't open client_key $conf{key_file}: $!"); - my $digest = join("", ); + my $digest = ; + chomp $digest; close(FILE); - - #my $digest = "124567890"; my ($status, $message) = parse_response(send_command("IDENTIFY", $digest)); unless (defined $status && $status eq "OK") { print "Could not identify to server: $message\n"; -- cgit v1.2.3-65-gdbad From 7beafd3d7f98786a99bb7c80899f11db58c7ae0a Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Tue, 1 Jan 2008 02:37:08 +0000 Subject: modify get_jobs query to make sure to get the latest job_history entry for the job svn path=/branches/new-fu/; revision=304 --- server/scireserver.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 4de6362..fd2da34 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -208,7 +208,7 @@ sub get_jobs { AND jobs.jobid = jobs_clients.jobid AND (job_conditions.deploy_time < now()) AND (job_conditions.expiration_time > now()) - AND job_history.statusid = '?' + AND (SELECT statusid FROM job_history WHERE jobid=jobs.jobid ORDER BY eventtime DESC LIMIT 1)=? ORDER BY jobs.priority,jobs.created EndOfQuery -- cgit v1.2.3-65-gdbad From 4c8c114b137aa40b4a9b75857922e7b671df9311 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Tue, 1 Jan 2008 03:08:59 +0000 Subject: fully implement get_jobs svn path=/branches/new-fu/; revision=305 --- server/scireserver.pl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index fd2da34..d8e27c3 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -75,8 +75,8 @@ while(<>) { } if ($command eq "GET_JOBS") { - get_jobs(); - + my @jobs = get_jobs(); + print "OK " . join(",", @jobs) . "\n"; } elsif ($command eq "GET_JOB") { my $job = $args[0]; get_job($job); @@ -108,7 +108,6 @@ sub read_config_file { debug("Conf file $conf_file read."); } - #New clients must be registered so they can be given a key to use (perhaps for job file transfers?) for authentication. This must be allowed before identifying. sub register_client { my ($mac,$ip, $hostname) = @_; @@ -174,7 +173,6 @@ sub register_client { print "OK $digest\n"; } - #Identify the client by looking up the fingerprint in the database, and matching it up. sub identify_client { my $digest = shift; @@ -199,7 +197,6 @@ sub identify_client { } - sub get_jobs { my $query = <<'EndOfQuery'; SELECT jobs.jobid @@ -213,12 +210,14 @@ sub get_jobs { EndOfQuery debug("Query is $query"); - #$sth = $dbh->prepare($query); - #$sth->execute($client_id,$status_id); - #my $jobs_ref = $sth->fetchall_arrayref(); - #return $jobs_ref; - + $sth = $dbh->prepare($query); + $sth->execute($client_id,$status_id); + my $jobs_ref = $sth->fetchall_arrayref(); + # Don't ask me...ask the guys in #perl :P + my @jobs = map { @$_ } @$jobs_ref; + return @jobs; } + sub get_job { my $jobid = shift; #Validate your inputs! @@ -239,6 +238,7 @@ sub get_job { my $filename = "$conf{job_dir}/$client_id/queue/$jobid.job"; return "OK $filename\n"; } + sub set_job_status { my ($jobid,$status) = @_; #Validate your inputs! -- cgit v1.2.3-65-gdbad From 38b2f34b140cf6497f3836a1d605926f1fc33247 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Tue, 1 Jan 2008 03:14:28 +0000 Subject: add status_id param to get_jobs() svn path=/branches/new-fu/; revision=306 --- server/scireserver.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index d8e27c3..9329f5c 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -198,6 +198,7 @@ sub identify_client { } sub get_jobs { + my $status_id = shift; my $query = <<'EndOfQuery'; SELECT jobs.jobid FROM jobs NATURAL JOIN jobs_clients NATURAL JOIN job_conditions NATURAL JOIN job_history @@ -211,7 +212,7 @@ EndOfQuery debug("Query is $query"); $sth = $dbh->prepare($query); - $sth->execute($client_id,$status_id); + $sth->execute($client_id, $status_id); my $jobs_ref = $sth->fetchall_arrayref(); # Don't ask me...ask the guys in #perl :P my @jobs = map { @$_ } @$jobs_ref; -- cgit v1.2.3-65-gdbad From 9be3d2375b5347d420e69acd8201762dd0a2608c Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 2 Jan 2008 02:01:51 +0000 Subject: add debug to response line added other paths to check_job_dir fixed up get_job a bit. svn path=/branches/new-fu/; revision=310 --- client/scireclient.pl | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 1da3dcd..640b3ec 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -95,6 +95,7 @@ sub send_command { #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! #if the server doesn't give you a newline this just hangs! my $response = ; + debug("Got response: ${response}"); return $response; } @@ -141,6 +142,21 @@ sub check_job_dir { mkpath( $conf{job_dir}, {verbose => 1, mode => 0660}) or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); } + if (! -d "$conf{job_dir}/queue") { + print "WARNING! $conf{job_dir}/queue does not exist...creating\n"; + mkpath( "$conf{job_dir}/queue", {verbose => 1, mode => 0660}) + or die("Couldn't make $conf{job_dir}/queue w/ perms 0660: $!"); + } + if (! -d "$conf{job_dir}/done") { + print "WARNING! $conf{job_dir}/done does not exist...creating\n"; + mkpath( "$conf{job_dir}/done", {verbose => 1, mode => 0660}) + or die("Couldn't make $conf{job_dir}/done w/ perms 0660: $!"); + } + if (! -d "$conf{job_dir}/failed") { + print "WARNING! $conf{job_dir}/failed does not exist...creating\n"; + mkpath( "$conf{job_dir}/failed", {verbose => 1, mode => 0660}) + or die("Couldn't make $conf{job_dir}/failed w/ perms 0660: $!"); + } } sub read_config_file { @@ -203,14 +219,11 @@ sub get_jobs { $jobs =~ s/\s//g; #Remove all whitespace my @jobs_list = split(/,/, $jobs); foreach my $job (@jobs_list) { - my ($status, $message) = parse_response(send_command("GET_JOB", $job)); - open(JOBFILE, ">$conf{job_dir}/queue/${job}.job") or do { - print "Could not open $conf{job_dir}/queue/${job}.job for writing: $!"; - next; - }; + my ($status, $filename) = parse_response(send_command("GET_JOB", $job)); + #SCP the file to $conf{job_dir}/queue/ + + system("cp $filename $conf{job_dir}/queue/") and die("Can't copy file: $!"); #Temporary hack. only works locally. # XXX: Modify this to fetch a file instead -# print JOBFILE parse_response($resp); - close(JOBFILE); debug("Fetched job $job "); } #This function doesn't actually need to do anything with the list of jobs, the executor handles that part. -- cgit v1.2.3-65-gdbad From f5f0de98906efb62a9376c0504f4320426b96d5a Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 2 Jan 2008 02:03:43 +0000 Subject: fixed get_jobs. coded get_job. but there seems to be a problem with it. commented out most of it for now, still doesn't work. svn path=/branches/new-fu/; revision=311 --- server/scireserver.pl | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 9329f5c..8e49ee7 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -7,6 +7,8 @@ use warnings; use DBI; use Data::Dumper; use Digest::MD5 qw(md5 md5_hex ); +use File::Path; +use XML::Dumper; $| = 1; @@ -198,21 +200,23 @@ sub identify_client { } sub get_jobs { - my $status_id = shift; + + #FIXME expand jobs for $client_id + my $query = <<'EndOfQuery'; SELECT jobs.jobid - FROM jobs NATURAL JOIN jobs_clients NATURAL JOIN job_conditions NATURAL JOIN job_history + FROM jobs NATURAL JOIN jobs_clients NATURAL JOIN job_conditions WHERE jobs_clients.clientid = ? AND jobs.jobid = jobs_clients.jobid AND (job_conditions.deploy_time < now()) - AND (job_conditions.expiration_time > now()) - AND (SELECT statusid FROM job_history WHERE jobid=jobs.jobid ORDER BY eventtime DESC LIMIT 1)=? + AND ((job_conditions.expiration_time > now()) OR (job_conditions.expiration_time IS NULL)) ORDER BY jobs.priority,jobs.created EndOfQuery + #FIXME ADD JOB DEPENDENCIES TO THIS QUERY. debug("Query is $query"); - $sth = $dbh->prepare($query); - $sth->execute($client_id, $status_id); + my $sth = $dbh->prepare($query); + $sth->execute($client_id); my $jobs_ref = $sth->fetchall_arrayref(); # Don't ask me...ask the guys in #perl :P my @jobs = map { @$_ } @$jobs_ref; @@ -224,19 +228,31 @@ sub get_job { #Validate your inputs! my $query = 'SELECT * FROM jobs LEFT JOIN job_conditions on (jobs.jobid) WHERE jobs.jobid = ?'; debug("Query is $query"); - #my $sth = $dbh->prepare($query); - #$sth->execute($jobid); - #my $job = $sth->fetchrow_hashref(); - #my $scriptid = $job{'script'}; +# my $sth = $dbh->prepare($query); +# $sth->execute($jobid); +# my $job = $sth->fetchrow_hashref(); +# my $scriptid = $job->{'script'}; $query = 'SELECT * FROM scripts WHERE scriptid=?'; debug("Query is $query"); - #$sth = $dbh->prepare($query); - #$sth->execute($scriptid); - #$job{'script'} = $sth->fetchrow_hashref(); +# $sth = $dbh->prepare($query); +# $sth->execute($scriptid); +# $job->{'script'} = $sth->fetchrow_hashref(); +# debug(Dumper($job)); #Write the job w/ all data to a jobfile with the following path /JOBDIR/CLIENT_ID/queue/JOBID.job - my $filename = "$conf{job_dir}/$client_id/queue/$jobid.job"; + my $path = "$conf{job_dir}/$client_id/queue"; + my $filename = "$path/$jobid.job"; +# unless (-d $path) { +# print "WARNING! $path does not exist...creating\n"; +# mkpath( $path, {verbose => 1, mode => 0660}) +# or die("Couldn't make $path w/ perms 0660: $!"); +# } +# open(FH, ">$filename") or die("Couldn't open $filename: $!"); +# my $xml = pl2xml( $job ); +# print FH $xml."\n"; +# close(FH) or die("Couldn't close $filename : $!"); + debug("OK $filename\n"); return "OK $filename\n"; } -- cgit v1.2.3-65-gdbad From 15e7118dbd275fd3350435c2c049257630ceea38 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Wed, 2 Jan 2008 02:12:24 +0000 Subject: return job filename and print it svn path=/branches/new-fu/; revision=312 --- server/scireserver.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 8e49ee7..9e20154 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -81,8 +81,8 @@ while(<>) { print "OK " . join(",", @jobs) . "\n"; } elsif ($command eq "GET_JOB") { my $job = $args[0]; - get_job($job); - + my $jobfile = get_job($job); + print "OK ${jobfile}\n"; } elsif ($command eq "SET_JOB_STATUS") { my ($jobid,$status) = @args; set_job_status($jobid,$status); @@ -253,7 +253,7 @@ sub get_job { # print FH $xml."\n"; # close(FH) or die("Couldn't close $filename : $!"); debug("OK $filename\n"); - return "OK $filename\n"; + return $filename; } sub set_job_status { -- cgit v1.2.3-65-gdbad From ffe274254918548d365926f58c80e93ba69d897e Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 2 Jan 2008 04:16:33 +0000 Subject: fixin up get_jobs a bit. svn path=/branches/new-fu/; revision=313 --- client/scireclient.pl | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 640b3ec..15f323e 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -203,7 +203,7 @@ sub identify_client { close(FILE); my ($status, $message) = parse_response(send_command("IDENTIFY", $digest)); unless (defined $status && $status eq "OK") { - print "Could not identify to server: $message\n"; + print "ERROR Could not identify to server: $message\n"; return 0; } debug("Client identified"); @@ -213,20 +213,27 @@ sub identify_client { sub get_jobs { my ($status, $jobs) = parse_response(send_command("GET_JOBS")); unless (defined $status && $status eq "OK") { - print "Could not get jobs list from server: $jobs\n"; + print "Could not get jobs list from server: $status\n"; return 0; } - $jobs =~ s/\s//g; #Remove all whitespace - my @jobs_list = split(/,/, $jobs); - foreach my $job (@jobs_list) { - my ($status, $filename) = parse_response(send_command("GET_JOB", $job)); - #SCP the file to $conf{job_dir}/queue/ + if (defined($jobs) && $jobs) { + $jobs =~ s/\s//g; #Remove all whitespace + my @jobs_list = split(/,/, $jobs); + foreach my $job (@jobs_list) { + my ($status, $filename) = parse_response(send_command("GET_JOB", $job)); + #SCP the file to $conf{job_dir}/queue/ - system("cp $filename $conf{job_dir}/queue/") and die("Can't copy file: $!"); #Temporary hack. only works locally. - # XXX: Modify this to fetch a file instead - debug("Fetched job $job "); + system("cp $filename $conf{job_dir}/queue/") and die("Can't copy file: $!"); #Temporary hack. only works locally. + # XXX: Modify this to fetch a file instead + debug("Fetched job $job "); + my ($status2,$message) = parse_response(send_command("JOB_FETCHED", $job)); + unless (defined $status2 && $status2 eq "OK") { + die("ERROR Could not signal job was fetched: $message\n"); + } + + } + #This function doesn't actually need to do anything with the list of jobs, the executor handles that part. } - #This function doesn't actually need to do anything with the list of jobs, the executor handles that part. } sub scan_jobs_dir { -- cgit v1.2.3-65-gdbad From a086bd72dbbcd8c027c4daafa3a7b9782e509c0f Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 2 Jan 2008 04:18:50 +0000 Subject: updating this. minor fixes. adding job_fetched code. svn path=/branches/new-fu/; revision=314 --- server/scireserver.pl | 77 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 9e20154..ba864f5 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -83,9 +83,12 @@ while(<>) { my $job = $args[0]; my $jobfile = get_job($job); print "OK ${jobfile}\n"; + } elsif ($command eq "JOB_FETCHED") { + my $job = $args[0]; + job_fetched($job) and print "OK\n"; } elsif ($command eq "SET_JOB_STATUS") { my ($jobid,$status) = @args; - set_job_status($jobid,$status); + set_job_status($jobid,$status) and print "OK\n"; } else { print "ERROR The command $command is unknown. Please try again.\n"; @@ -228,37 +231,73 @@ sub get_job { #Validate your inputs! my $query = 'SELECT * FROM jobs LEFT JOIN job_conditions on (jobs.jobid) WHERE jobs.jobid = ?'; debug("Query is $query"); -# my $sth = $dbh->prepare($query); -# $sth->execute($jobid); -# my $job = $sth->fetchrow_hashref(); -# my $scriptid = $job->{'script'}; + my $sth = $dbh->prepare($query); + $sth->execute($jobid); + my $job = $sth->fetchrow_hashref(); + my $scriptid = $job->{'script'}; $query = 'SELECT * FROM scripts WHERE scriptid=?'; debug("Query is $query"); -# $sth = $dbh->prepare($query); -# $sth->execute($scriptid); -# $job->{'script'} = $sth->fetchrow_hashref(); + $sth = $dbh->prepare($query); + $sth->execute($scriptid); + $job->{'script'} = $sth->fetchrow_hashref(); -# debug(Dumper($job)); + debug(Dumper($job)); #Write the job w/ all data to a jobfile with the following path /JOBDIR/CLIENT_ID/queue/JOBID.job my $path = "$conf{job_dir}/$client_id/queue"; my $filename = "$path/$jobid.job"; -# unless (-d $path) { -# print "WARNING! $path does not exist...creating\n"; -# mkpath( $path, {verbose => 1, mode => 0660}) -# or die("Couldn't make $path w/ perms 0660: $!"); -# } -# open(FH, ">$filename") or die("Couldn't open $filename: $!"); -# my $xml = pl2xml( $job ); -# print FH $xml."\n"; -# close(FH) or die("Couldn't close $filename : $!"); - debug("OK $filename\n"); + unless (-d $path) { + print "WARNING! $path does not exist...creating\n"; + mkpath( $path, {verbose => 1, mode => 0660}) + or die("Couldn't make $path w/ perms 0660: $!"); + } + open(FH, ">$filename") or die("Couldn't open $filename: $!"); + my $xml = pl2xml( $job ); + print FH $xml."\n"; + close(FH) or die("Couldn't close $filename : $!"); + debug("OK $filename"); return $filename; } +sub job_fetched { + my $jobid = shift; + set_job_status($jobid,'Downloaded') or print "ERROR could not set job status to downloaded.\n"; + + eval { + my $query = 'DELETE FROM jobs_clients WHERE jobid=? AND clientid=?'; + debug("Query is $query"); + my $sth = $dbh->prepare($query); + $sth->execute($jobid,$client_id); + }; + ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; + return 1; +} + sub set_job_status { my ($jobid,$status) = @_; #Validate your inputs! + $jobid =~ /^\d+$/ or die("Invalid jobid $jobid"); + #fixme validate status + my $status_id; + eval { + my $query = 'SELECT statusid FROM jobs_status WHERE statusname = ?'; + debug("Query is $query"); +# $status_id = "4"; #db.conn.GetRow($query) + my $sth = $dbh->prepare($query); + $sth->execute($status); + $status_id = $sth->fetchrow_hashref->{'statusid'}; + }; + ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; + $status_id or print "ERROR Invalid status id $status_id\n"; + + eval { + my $query = 'INSERT INTO job_history (jobid,clientid,statusid) VALUES (?,?,?)'; + debug("Query is $query"); + my $sth = $dbh->prepare($query); + $sth->execute($jobid,$client_id,$status_id); + }; + ($@) and print "ERROR Could not insert into job_history: $DBI::errstr\n"; + return 1; } sub parse_command { -- cgit v1.2.3-65-gdbad From f72beed61d7c018e69ba167bb41204d142a466d4 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Wed, 2 Jan 2008 04:24:40 +0000 Subject: fix parse_response() regex to ignore trailing whitespace svn path=/branches/new-fu/; revision=315 --- client/scireclient.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 15f323e..25e6c9b 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -101,7 +101,7 @@ sub send_command { sub parse_response { my $response = shift; - $response =~ /^(OK|ERROR)(?: (.+))?$/; + $response =~ /^(OK|ERROR)(?: (.+?))?\s*$/; my ($status, $message) = ($1, $2); return ($status, $message); } -- cgit v1.2.3-65-gdbad From 6f4f0c362cb59f83c9209c58eeb663f52bc655bb Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Wed, 2 Jan 2008 04:36:58 +0000 Subject: remove the jobfile once it's been successfully sent to the client. svn path=/branches/new-fu/; revision=316 --- server/scireserver.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/scireserver.pl b/server/scireserver.pl index ba864f5..835a10d 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -270,6 +270,9 @@ sub job_fetched { $sth->execute($jobid,$client_id); }; ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; + + my $filename = "$conf{job_dir}/$client_id/queue/$jobid.job"; + unlink ($filename) or die("ERROR Could not unlink the jobfile from the queue. filename: $filename : $!"); return 1; } -- cgit v1.2.3-65-gdbad From 60ec495aaa37ee1836998ae8479a95c7f04fd2ef Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Thu, 3 Jan 2008 03:55:50 +0000 Subject: adding job expansion code. svn path=/branches/new-fu/; revision=317 --- server/scireserver.pl | 105 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 835a10d..b50e462 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -88,7 +88,7 @@ while(<>) { job_fetched($job) and print "OK\n"; } elsif ($command eq "SET_JOB_STATUS") { my ($jobid,$status) = @args; - set_job_status($jobid,$status) and print "OK\n"; + set_job_status($jobid,$client_id,$status) and print "OK\n"; } else { print "ERROR The command $command is unknown. Please try again.\n"; @@ -205,6 +205,7 @@ sub identify_client { sub get_jobs { #FIXME expand jobs for $client_id + expand_jobs(); my $query = <<'EndOfQuery'; SELECT jobs.jobid @@ -261,7 +262,7 @@ sub get_job { sub job_fetched { my $jobid = shift; - set_job_status($jobid,'Downloaded') or print "ERROR could not set job status to downloaded.\n"; + set_job_status($jobid,$client_id,'Downloaded', 'Job downloaded by client.') or print "ERROR could not set job status to downloaded.\n"; eval { my $query = 'DELETE FROM jobs_clients WHERE jobid=? AND clientid=?'; @@ -277,9 +278,12 @@ sub job_fetched { } sub set_job_status { - my ($jobid,$status) = @_; + my ($jobid,$id_of_client,$status,$eventmsg) = @_; #Validate your inputs! $jobid =~ /^\d+$/ or die("Invalid jobid $jobid"); + $id_of_client ||= $client_id; + $id_of_client =~ /^\d+$/ or die("Invalid id of client $id_of_client"); + $eventmsg ||= "Server status update."; #fixme validate status my $status_id; eval { @@ -294,10 +298,10 @@ sub set_job_status { $status_id or print "ERROR Invalid status id $status_id\n"; eval { - my $query = 'INSERT INTO job_history (jobid,clientid,statusid) VALUES (?,?,?)'; + my $query = 'INSERT INTO job_history (jobid,clientid,statusid,eventmsg) VALUES (?,?,?,?)'; debug("Query is $query"); my $sth = $dbh->prepare($query); - $sth->execute($jobid,$client_id,$status_id); + $sth->execute($jobid,$id_of_client,$status_id,$eventmsg); }; ($@) and print "ERROR Could not insert into job_history: $DBI::errstr\n"; return 1; @@ -313,3 +317,94 @@ sub parse_command { } return @parts; } + +sub expand_jobs { + #Searches for the group jobs that the client must be into and does the expansion. + my @groups = get_client_groups(); + foreach my $groupid (@groups) { + my @members = get_group_clients($groupid); + eval { + my $query = <<'EndOfQuery2'; +SELECT DISTINCT(jobs_clients.jobid) +FROM jobs_clients LEFT JOIN job_conditions on (jobs_clients.jobid=job_conditions.jobid) +WHERE jobs_clients.groupid = ? +AND (job_conditions.deploy_time < now()) +AND (job_conditions.expiration_time > now()) +AND job_conditions.last_run_date < job_conditions.deploy_time +EndOfQuery2 + my $sth = $dbh->prepare($query); + $sth->execute($groupid); + # $dbh->do('LOCK TABLES `jobs_clients` WRITE, `job_conditions` WRITE, `job_history` WRITE'); + while( my $jobid = $sth->fetchrow_hashref->{'jobid'} ) { + foreach my $member (@members) { + $query = 'INSERT INTO jobs_clients (jobid, clientid) VALUES (?,?)'; + debug("Query is $query"); + my $sth2 = $dbh->prepare($query); + $sth2->execute($jobid,$member); + + set_job_status($jobid,$member,'Pending', 'Job expanded.') or print "ERROR could not add expanded jobs to job_history.\n"; + } + $query = 'UPDATE `job_conditions` SET last_run_date = now() WHERE jobid = ?'; + debug("Query is $query"); + my $sth3 = $dbh->prepare($query); + $sth3->execute($jobid); + + } + # $dbh->do('UNLOCK TABLES'); + }; + ($@) and print "ERROR Could not expand jobs: $DBI::errstr\n"; + return undef; + } +} + +######################################################### +# PHPGACL FUNCTIONS +######################################################### + +sub get_client_groups { + my $query; + my $option = 'NO RECURSE'; + # If RECURSE it will get all ancestor groups. defaults to only get direct parents. + + debug("get_object_groups(): Object ID: $client_id, option: $option"); + my $object_type = 'axo'; + my $group_table = 'gacl_axo_groups'; + my $map_table = 'gacl_groups_axo_map'; + + if ($option eq 'RECURSE') { + $query = "SELECT DISTINCT g.id as group_id FROM $map_table gm "; + $query .= "LEFT JOIN $group_table g1 ON g1.id=gm.group_id "; + $query .= "LEFT JOIN $group_table g ON g.lft<=g1.lft AND g.rgt>=g1.rgt"; + } else { + $query = "SELECT gm.group_id FROM $map_table gm "; + } + $query .= " WHERE gm.axo_id=?"; + debug("Query is $query"); + eval { + my $sth = $dbh->prepare($query); + $sth->execute($client_id); + my $groups_ref = $sth->fetchall_arrayref(); + # Don't ask me...ask the guys in #perl :P + my @groups = map { @$_ } @$groups_ref; + return @groups; + }; + ($@) and print "ERROR Could not get client groups: $DBI::errstr\n"; + return undef; +} + +sub get_group_clients { + #This function gets the members of groups. Returns an array containing those clients, empty otherwise. + my $groupid = shift; + my $query = 'SELECT axo_id FROM gacl_groups_axo_map WHERE group_id = ?'; + debug("Query is $query"); + eval { + my $sth = $dbh->prepare($query); + $sth->execute($groupid); + my $members_ref = $sth->fetchall_arrayref(); + # Don't ask me...ask the guys in #perl :P + my @members = map { @$_ } @$members_ref; + return @members; + }; + ($@) and print "ERROR Could not get group members: $DBI::errstr\n"; + return undef; +} -- cgit v1.2.3-65-gdbad From ce9d45f4462b255b2981afd4f2ff33d3c93e08fb Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sat, 5 Jan 2008 03:02:34 +0000 Subject: fixed up the expansion w/ one last query. svn path=/branches/new-fu/; revision=318 --- server/scireserver.pl | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index b50e462..db05d43 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -322,6 +322,7 @@ sub expand_jobs { #Searches for the group jobs that the client must be into and does the expansion. my @groups = get_client_groups(); foreach my $groupid (@groups) { + debug("Groupid is $groupid"); my @members = get_group_clients($groupid); eval { my $query = <<'EndOfQuery2'; @@ -329,13 +330,15 @@ SELECT DISTINCT(jobs_clients.jobid) FROM jobs_clients LEFT JOIN job_conditions on (jobs_clients.jobid=job_conditions.jobid) WHERE jobs_clients.groupid = ? AND (job_conditions.deploy_time < now()) -AND (job_conditions.expiration_time > now()) -AND job_conditions.last_run_date < job_conditions.deploy_time +AND ((job_conditions.expiration_time > now()) OR (job_conditions.expiration_time IS NULL)) +AND ((job_conditions.last_run_date < job_conditions.deploy_time) OR (job_conditions.last_run_date IS NULL)) EndOfQuery2 + debug("Query is $query"); my $sth = $dbh->prepare($query); $sth->execute($groupid); - # $dbh->do('LOCK TABLES `jobs_clients` WRITE, `job_conditions` WRITE, `job_history` WRITE'); - while( my $jobid = $sth->fetchrow_hashref->{'jobid'} ) { + $dbh->do('LOCK TABLES `jobs_clients` WRITE, `job_conditions` WRITE, `job_history` WRITE, `jobs_status` WRITE'); + while( my $jobref = $sth->fetchrow_hashref() ) { + my $jobid = $jobref->{'jobid'}; foreach my $member (@members) { $query = 'INSERT INTO jobs_clients (jobid, clientid) VALUES (?,?)'; debug("Query is $query"); @@ -349,10 +352,16 @@ EndOfQuery2 my $sth3 = $dbh->prepare($query); $sth3->execute($jobid); + # One last query to remove the row from jobs_clients so someone else doesn't expand it. + $query = 'DELETE FROM `jobs_clients` WHERE groupid=? AND jobid=?'; + debug("Query is $query"); + my $sth4 = $dbh->prepare($query); + $sth4->execute($groupid,$jobid); } - # $dbh->do('UNLOCK TABLES'); + + $dbh->do('UNLOCK TABLES'); }; - ($@) and print "ERROR Could not expand jobs: $DBI::errstr\n"; + ($@) and print "ERROR Could not expand jobs: $@ $DBI::errstr\n"; return undef; } } @@ -363,6 +372,7 @@ EndOfQuery2 sub get_client_groups { my $query; + my @groups; my $option = 'NO RECURSE'; # If RECURSE it will get all ancestor groups. defaults to only get direct parents. @@ -385,16 +395,16 @@ sub get_client_groups { $sth->execute($client_id); my $groups_ref = $sth->fetchall_arrayref(); # Don't ask me...ask the guys in #perl :P - my @groups = map { @$_ } @$groups_ref; - return @groups; + @groups = map { @$_ } @$groups_ref; }; ($@) and print "ERROR Could not get client groups: $DBI::errstr\n"; - return undef; + return @groups; } sub get_group_clients { #This function gets the members of groups. Returns an array containing those clients, empty otherwise. my $groupid = shift; + my @members; my $query = 'SELECT axo_id FROM gacl_groups_axo_map WHERE group_id = ?'; debug("Query is $query"); eval { @@ -402,9 +412,8 @@ sub get_group_clients { $sth->execute($groupid); my $members_ref = $sth->fetchall_arrayref(); # Don't ask me...ask the guys in #perl :P - my @members = map { @$_ } @$members_ref; - return @members; + @members = map { @$_ } @$members_ref; }; ($@) and print "ERROR Could not get group members: $DBI::errstr\n"; - return undef; + return @members; } -- cgit v1.2.3-65-gdbad From d8c52e5e9a848bab9e77a4db777027c3d710e57e Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 03:34:14 +0000 Subject: initial commit of Scire.pm svn path=/branches/new-fu/; revision=320 --- client/Scire.pm | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 client/Scire.pm diff --git a/client/Scire.pm b/client/Scire.pm new file mode 100644 index 0000000..8b94082 --- /dev/null +++ b/client/Scire.pm @@ -0,0 +1,37 @@ +package Scire::Job; + +sub new { + my $proto = shift; + my $class = ref($proto) || $proto; + my $filename = shift; + my $self = {}; + bless ($self, $class); + if(defined $filename) { + $self->set_filename($filename); + } + return $self; +} + +sub set_filename { + my $self = shift; + my $filename = shift; + $self->{filename} = $filename; + my $jobcontents; + my $jobdata; + open JOB, "< ${filename}" or die "Can't open file ${filename}"; + $jobcontents = join("", ); + close JOB; + $jobdata = eval($jobcontents); + ($@) and print "ERROR: Could not parse job file ${filename}!\n"; + if(defined $jobdata->{script}) { + for(keys %{$jobdata->{script}}) { + $self->{$_} = $jobdata->{script}->{$_}; + } + } + for(keys %{$jobdata}) { + $self->{$_} = $jobdata->{$_}; + } +} + +1; + -- cgit v1.2.3-65-gdbad From 60157f5a92484026f72b2ce198b25839a016421c Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 03:40:52 +0000 Subject: don't stick 'script' in the hash svn path=/branches/new-fu/; revision=321 --- client/Scire.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/Scire.pm b/client/Scire.pm index 8b94082..8ae5973 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -29,7 +29,7 @@ sub set_filename { } } for(keys %{$jobdata}) { - $self->{$_} = $jobdata->{$_}; + $self->{$_} = $jobdata->{$_} unless($_ eq "script"); } } -- cgit v1.2.3-65-gdbad From c7bf256c204118beb9f207e6f487fc7f53e00a2f Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 04:19:08 +0000 Subject: switch back to Data::Dumper for job output svn path=/branches/new-fu/; revision=322 --- server/scireserver.pl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index db05d43..ca256bd 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -8,7 +8,6 @@ use DBI; use Data::Dumper; use Digest::MD5 qw(md5 md5_hex ); use File::Path; -use XML::Dumper; $| = 1; @@ -253,8 +252,8 @@ sub get_job { or die("Couldn't make $path w/ perms 0660: $!"); } open(FH, ">$filename") or die("Couldn't open $filename: $!"); - my $xml = pl2xml( $job ); - print FH $xml."\n"; + my $jobdata = Dumper($job); + print FH $jobdata . "\n"; close(FH) or die("Couldn't close $filename : $!"); debug("OK $filename"); return $filename; -- cgit v1.2.3-65-gdbad From 0ec06099df539205d8561236a5152ffcf8c60db8 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 04:23:42 +0000 Subject: set Data::Dumper::Purity to 1 for nested data svn path=/branches/new-fu/; revision=323 --- server/scireserver.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/server/scireserver.pl b/server/scireserver.pl index ca256bd..e3d201b 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -10,6 +10,7 @@ use Digest::MD5 qw(md5 md5_hex ); use File::Path; $| = 1; +$Data::Dumper::Purity = 1; my $ETC_DIR = "/etc/scire"; my $SCIRE_CONFIG_FILE = "${ETC_DIR}/scireserver.conf"; -- cgit v1.2.3-65-gdbad From 6fd559d2fb4ed1db2a3adb65b1fbd4ff7ddbad5a Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 04:31:34 +0000 Subject: move bits of the server communication into Scire.pm svn path=/branches/new-fu/; revision=324 --- client/Scire.pm | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ client/scireclient.pl | 82 +++++++----------------------------------------- 2 files changed, 97 insertions(+), 71 deletions(-) diff --git a/client/Scire.pm b/client/Scire.pm index 8ae5973..5530ff3 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -33,5 +33,91 @@ sub set_filename { } } +package Scire::Communicator; + +use IPC::Open2; + +my ($SERVER_STDIN, $SERVER_STDOUT); + +sub new { + my $proto = shift; + my $class = ref($proto) || $proto; + my $self = { + port => 22, + user => scire, + server_script => "/usr/bin/scireserver.pl", +# SERVER_STDOUT => undef, +# SERVER_STDIN => undef, + @_ + }; + bless ($self, $class); + $self->build_connection_command(); + return $self; +} + +sub send_command { + my $self = shift; + my $cmd = shift; + my @args = @_; + my $tosend = "${cmd}"; + + for my $arg (@args) { + if($arg =~ /^[0-9]+$/) { + $tosend .= " ${arg}"; + } else { + $arg =~ s/"/\\"/g; + $tosend .= " \"${arg}\""; + } + } +# debug("Sending: ${tosend}"); + print SERVER_STDIN "${tosend}\n"; + #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! + #if the server doesn't give you a newline this just hangs! + my $response = ; +# debug("Got response: ${response}"); + return $self->parse_response($response); +} + +sub parse_response { + my $self = shift; + my $response = shift; + $response =~ /^(OK|ERROR)(?: (.+?))?\s*$/; + my ($status, $message) = ($1, $2); + return ($status, $message); +} + +sub create_connection { + my $self = shift; + # XXX: How do we capture this error? $pid has a valid value even if the + # process fails to run, since it just returns the PID of the forked perl + # process. I tried adding 'or die' after it, but it didn't help since it + # doesn't fail in the main process. When it fails, it outputs an error + # to STDERR: + # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 + $self->{connection_pid} = open2(*SERVER_STDOUT, *SERVER_STDIN, $self->{connection_command}); +} + +sub build_connection_command { + my $self = shift; + # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" + my $connection_command = "ssh "; + $connection_command .= "-o BatchMode yes "; + $connection_command .= "-o SendEnv 'SCIRE_*' "; + $connection_command .= "-o ServerAliveInterval 15 -o ServerAliveCountMax 4 "; + if(defined($self->{port})) { + $connection_command .= "-o Port=$conf{port} "; + } + $connection_command .= "$self->{user}\@$self->{host} $self->{server_script}"; + + if (-d ".svn") { + # Overwrite $connection_command in the case of a dev environment for now + $connection_command = "../server/scireserver.pl"; + } + +# return $connection_command; + $self->{connection_command} = $connection_command; +} + + 1; diff --git a/client/scireclient.pl b/client/scireclient.pl index 25e6c9b..0925f07 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -5,7 +5,7 @@ use strict; use warnings; -use IPC::Open2; +use Scire; use Getopt::Long; use Data::Dumper; use File::Path; @@ -15,7 +15,7 @@ my $ETC_DIR = "/etc/scire"; my $SCIRE_CONFIG_FILE = "${ETC_DIR}/scire.conf"; my %conf; my ($SERVER_STDOUT, $SERVER_STDIN); -my $connection_pid; +my $comm; run_main(); @@ -26,11 +26,10 @@ sub run_main { check_job_dir(); - my $connection_command = build_connection_command(); - #ok folks so here's how this thang goes down. #1. Connect. - create_connection($connection_command); + $comm = Scire::Communicator->new( host => $conf{host}, user => $conf{user}, port => $conf{port} ); + $comm->create_connection(); #2. Register with the DB. (only it knows if you're allowed to be active) # If we do not have a defined key file, we assume this is the first run of this client @@ -77,65 +76,6 @@ sub parse_command_line { } -sub send_command { - my $cmd = shift; - my @args = @_; - my $tosend = "${cmd}"; - - for my $arg (@args) { - if($arg =~ /^[0-9]+$/) { - $tosend .= " ${arg}"; - } else { - $arg =~ s/"/\\"/g; - $tosend .= " \"${arg}\""; - } - } - debug("Sending: ${tosend}"); - print SERVER_STDIN "${tosend}\n"; - #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! - #if the server doesn't give you a newline this just hangs! - my $response = ; - debug("Got response: ${response}"); - return $response; -} - -sub parse_response { - my $response = shift; - $response =~ /^(OK|ERROR)(?: (.+?))?\s*$/; - my ($status, $message) = ($1, $2); - return ($status, $message); -} - -sub create_connection { - # XXX: How do we capture this error? $pid has a valid value even if the - # process fails to run, since it just returns the PID of the forked perl - # process. I tried adding 'or die' after it, but it didn't help since it - # doesn't fail in the main process. When it fails, it outputs an error - # to STDERR: - # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 - my $connection_command = shift; - $connection_pid = open2(*SERVER_STDOUT, *SERVER_STDIN, $connection_command); -} - -sub build_connection_command { - # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" - my $connection_command = "ssh "; - $connection_command .= "-o BatchMode yes "; - $connection_command .= "-o SendEnv 'SCIRE_*' "; - $connection_command .= "-o ServerAliveInterval 15 -o ServerAliveCountMax 4 "; - if(defined($conf{port})) { - $connection_command .= "-o Port=$conf{port} "; - } - $connection_command .= "$conf{user}\@$conf{host} $conf{server_script}"; - - if (-d ".svn") { - # Overwrite $connection_command in the case of a dev environment for now - $connection_command = "../server/scireserver.pl"; - } - - return $connection_command; -} - sub check_job_dir { if (! -d $conf{job_dir}) { print "WARNING! $conf{job_dir} does not exist...creating\n"; @@ -188,7 +128,7 @@ sub register_client { # my $ip = "192.168.2.3"; my ($mac, $ip) = get_interface_info(defined $conf{interface} && $conf{interface} ? $conf{interface} : "eth0"); my $hostname = hostname(); - my ($status, $message) = parse_response(send_command("REGISTER", $mac, $ip, $hostname)); + my ($status, $message) = $comm->send_command("REGISTER", $mac, $ip, $hostname); die "Could not register client $mac w/ ip $ip and hostname $hostname. Got: $message" if (! defined $status or $status ne "OK"); debug("Client registered. Status is pending. digest is $message"); open(FILE, ">$conf{key_file}") or die("Couldn't open key file $conf{key_file} for writing: $!"); @@ -201,7 +141,7 @@ sub identify_client { my $digest = ; chomp $digest; close(FILE); - my ($status, $message) = parse_response(send_command("IDENTIFY", $digest)); + my ($status, $message) = $comm->send_command("IDENTIFY", $digest); unless (defined $status && $status eq "OK") { print "ERROR Could not identify to server: $message\n"; return 0; @@ -211,7 +151,7 @@ sub identify_client { } sub get_jobs { - my ($status, $jobs) = parse_response(send_command("GET_JOBS")); + my ($status, $jobs) = $comm->send_command("GET_JOBS"); unless (defined $status && $status eq "OK") { print "Could not get jobs list from server: $status\n"; return 0; @@ -220,13 +160,13 @@ sub get_jobs { $jobs =~ s/\s//g; #Remove all whitespace my @jobs_list = split(/,/, $jobs); foreach my $job (@jobs_list) { - my ($status, $filename) = parse_response(send_command("GET_JOB", $job)); + my ($status, $filename) = $comm->send_command("GET_JOB", $job); #SCP the file to $conf{job_dir}/queue/ system("cp $filename $conf{job_dir}/queue/") and die("Can't copy file: $!"); #Temporary hack. only works locally. # XXX: Modify this to fetch a file instead debug("Fetched job $job "); - my ($status2,$message) = parse_response(send_command("JOB_FETCHED", $job)); + my ($status2,$message) = $comm->send_command("JOB_FETCHED", $job); unless (defined $status2 && $status2 eq "OK") { die("ERROR Could not signal job was fetched: $message\n"); } @@ -246,7 +186,7 @@ sub scan_jobs_dir { foreach my $job_file (@failed_jobs) { $job_file =~ /(\d+)\.job/; my $jobid = $1; - my ($status, $message) = parse_response(send_command("SET_JOB_STATUS $jobid 'Failed'")); + my ($status, $message) = $comm->send_command("SET_JOB_STATUS $jobid 'Failed'"); open(FILE, $job_file) or die "Couldn't open job file $job_file: $!"; my $job_data = join("", ); close(FILE); @@ -256,7 +196,7 @@ sub scan_jobs_dir { foreach my $job_file (@done_jobs) { $job_file =~ /(\d+)\.job/; my $jobid = $1; - my ($status, $message) = parse_response(send_command("SET_JOB_STATUS $jobid 'Done'")); + my ($status, $message) = $comm->send_command("SET_JOB_STATUS $jobid 'Done'"); } return @existing_jobs; -- cgit v1.2.3-65-gdbad From 1cf821f1dbe1a5fd28d24dc5b49efb082309259a Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 05:05:20 +0000 Subject: don't need SERVER_STDIN and SERVER_STDOUT here svn path=/branches/new-fu/; revision=325 --- client/scireclient.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index 0925f07..05d2340 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -14,7 +14,6 @@ use Sys::Hostname; my $ETC_DIR = "/etc/scire"; my $SCIRE_CONFIG_FILE = "${ETC_DIR}/scire.conf"; my %conf; -my ($SERVER_STDOUT, $SERVER_STDIN); my $comm; run_main(); -- cgit v1.2.3-65-gdbad From dbc8b62bc7b74495cee4f2ab9bf29c04ab78c165 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 05:15:06 +0000 Subject: move SERVER_STDIN and SERVER_STDOUT inside the class hash svn path=/branches/new-fu/; revision=326 --- client/Scire.pm | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/client/Scire.pm b/client/Scire.pm index 5530ff3..dca8da4 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -37,8 +37,6 @@ package Scire::Communicator; use IPC::Open2; -my ($SERVER_STDIN, $SERVER_STDOUT); - sub new { my $proto = shift; my $class = ref($proto) || $proto; @@ -46,8 +44,8 @@ sub new { port => 22, user => scire, server_script => "/usr/bin/scireserver.pl", -# SERVER_STDOUT => undef, -# SERVER_STDIN => undef, + SERVER_STDOUT => undef, + SERVER_STDIN => undef, @_ }; bless ($self, $class); @@ -69,12 +67,13 @@ sub send_command { $tosend .= " \"${arg}\""; } } -# debug("Sending: ${tosend}"); - print SERVER_STDIN "${tosend}\n"; + $tosend .= "\n"; + + my ($tmpin, $tmpout) = ($self->{SERVER_STDIN}, $self->{SERVER_STDOUT}); + print $tmpin $tosend; #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! #if the server doesn't give you a newline this just hangs! - my $response = ; -# debug("Got response: ${response}"); + my $response = <$tmpout>; return $self->parse_response($response); } @@ -94,7 +93,7 @@ sub create_connection { # doesn't fail in the main process. When it fails, it outputs an error # to STDERR: # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 - $self->{connection_pid} = open2(*SERVER_STDOUT, *SERVER_STDIN, $self->{connection_command}); + $self->{connection_pid} = open2($self->{SERVER_STDOUT}, $self->{SERVER_STDIN}, $self->{connection_command}); } sub build_connection_command { @@ -113,8 +112,6 @@ sub build_connection_command { # Overwrite $connection_command in the case of a dev environment for now $connection_command = "../server/scireserver.pl"; } - -# return $connection_command; $self->{connection_command} = $connection_command; } -- cgit v1.2.3-65-gdbad From f0d5d991f23122e323c720e8c7bcea5ec15471ba Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sat, 5 Jan 2008 05:15:51 +0000 Subject: whole bunch of updates here. first, created run_query and put the process of debug/prepare/execute into it. much simpler and cleaner! added RETURN_JOBFILE cmd and stub for JOBFILE_SENT. add a check to set_job_status to see if we're marking a job as finished. if so, it needs to do some more work. that part isn't coded yet. svn path=/branches/new-fu/; revision=327 --- server/scireserver.pl | 150 +++++++++++++++++++++++++++++--------------------- 1 file changed, 87 insertions(+), 63 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index e3d201b..68d6fe9 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -54,6 +54,7 @@ while(<>) { my ($command, @args) = parse_command($_); # chomp( my $line = $_); # debug("DEBUG: line is: $line"); +# SEE http://agaffney.org/mediawiki/index.php/SSH-based_protocol for documentation on the protocol. if($command eq "QUIT") { print "OK\n"; @@ -89,7 +90,14 @@ while(<>) { } elsif ($command eq "SET_JOB_STATUS") { my ($jobid,$status) = @args; set_job_status($jobid,$client_id,$status) and print "OK\n"; - + } elsif ($command eq "RETURN_JOBFILE") { + my $jobid = $args[0]; + my $filename = "$conf{job_dir}/$client_id/result/$jobid.result"; + print "OK ${filename}\n"; + } elsif ($command eq "JOBFILE_SENT") { + my $filename = $args[0]; + print "OK\n" and process_jobfile($filename); + } else { print "ERROR The command $command is unknown. Please try again.\n"; } @@ -127,50 +135,36 @@ sub register_client { eval { $query = 'SELECT statusid FROM client_status WHERE statusname = "Pending"'; - debug("Query is $query"); -# $status_id = "4"; #db.conn.GetRow($query) - $sth = $dbh->prepare($query); - $sth->execute(); + $sth = run_query($query); $status_id = $sth->fetchrow_hashref->{'statusid'}; }; ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; eval { - $query = 'LOCK TABLES `gacl_axo_seq` WRITE'; - debug("Query is $query"); + run_query('LOCK TABLES `gacl_axo_seq` WRITE'); +# debug("Query is $query"); #execute it - $dbh->do($query); +# $dbh->do($query); $query = 'SELECT id FROM `gacl_axo_seq`'; - debug("Query is $query"); - #$id = "56"; #execute $query - $sth = $dbh->prepare($query); - $sth->execute(); + $sth = run_query($query); $id = $sth->fetchrow_hashref->{'id'}; $id += 1; $query = 'UPDATE `gacl_axo_seq` SET id=?'; - debug("Query is $query"); - #execute with $id - $sth = $dbh->prepare($query); - $sth->execute($id); - $query = 'UNLOCK TABLES'; - debug("Query is $query"); - $dbh->do($query); + run_query($query,$id); + run_query('UNLOCK TABLES'); +# debug("Query is $query"); +# $dbh->do($query); }; ($@) and print "ERROR during fetching of id sequence: $DBI::errstr\n"; eval { $query = 'INSERT INTO `gacl_axo` (id,section_value,value,order_value,name,hidden) VALUES (?,"clients",?,"1",?,"0")'; - debug("Query is $query"); - $sth = $dbh->prepare($query); - $sth->execute($id, $hostname, $hostname); - #execute with $id, $hostname, $hostname + run_query($query,$id,$hostname,$hostname); #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. $query = 'INSERT INTO clients (clientid,digest,hostname,mac,ip,status) VALUES (?,?,?,?,?,?)'; - debug("Query is $query"); #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) - $sth = $dbh->prepare($query); - $sth->execute($id,$digest,$hostname,$mac,$ip,$status_id); + run_query($query,$id,$digest,$hostname,$mac,$ip,$status_id); }; ($@) and print "ERROR Could not insert client with $query: $DBI::errstr\n"; #FIXME look for "duplicate key" and if found fail and notify admin. @@ -186,14 +180,15 @@ sub identify_client { $digest =~ /^[A-Za-z0-9]+$/ or print "ERROR invalid digest!\n"; my $query = 'SELECT client_status.statusname, clients.clientid FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; - debug("Query is $query"); - my $sth = $dbh->prepare($query); - $sth->execute($digest); + my $sth = run_query($query,$digest); + #debug("Query is $query"); + #my $sth = $dbh->prepare($query); + #$sth->execute($digest); my $hashref = $sth->fetchrow_hashref(); debug(Dumper($hashref)); my $status_name = $hashref->{'statusname'}; $client_id = $hashref->{'clientid'}; - if ($client_id > 0) { #and ($status_name eq 'Active') { + if (defined($client_id) and $client_id > 0) { #and ($status_name eq 'Active') { $identified = 1; print "OK\n"; } else { @@ -218,9 +213,10 @@ sub get_jobs { EndOfQuery #FIXME ADD JOB DEPENDENCIES TO THIS QUERY. - debug("Query is $query"); - my $sth = $dbh->prepare($query); - $sth->execute($client_id); + my $sth = run_query($query,$client_id); +# debug("Query is $query"); +# my $sth = $dbh->prepare($query); +# $sth->execute($client_id); my $jobs_ref = $sth->fetchall_arrayref(); # Don't ask me...ask the guys in #perl :P my @jobs = map { @$_ } @$jobs_ref; @@ -231,16 +227,12 @@ sub get_job { my $jobid = shift; #Validate your inputs! my $query = 'SELECT * FROM jobs LEFT JOIN job_conditions on (jobs.jobid) WHERE jobs.jobid = ?'; - debug("Query is $query"); - my $sth = $dbh->prepare($query); - $sth->execute($jobid); + my $sth = run_query($query, $jobid); my $job = $sth->fetchrow_hashref(); my $scriptid = $job->{'script'}; $query = 'SELECT * FROM scripts WHERE scriptid=?'; - debug("Query is $query"); - $sth = $dbh->prepare($query); - $sth->execute($scriptid); + $sth = run_query($query,$scriptid); $job->{'script'} = $sth->fetchrow_hashref(); debug(Dumper($job)); @@ -266,9 +258,7 @@ sub job_fetched { eval { my $query = 'DELETE FROM jobs_clients WHERE jobid=? AND clientid=?'; - debug("Query is $query"); - my $sth = $dbh->prepare($query); - $sth->execute($jobid,$client_id); + run_query($query,$jobid,$client_id); }; ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; @@ -288,10 +278,7 @@ sub set_job_status { my $status_id; eval { my $query = 'SELECT statusid FROM jobs_status WHERE statusname = ?'; - debug("Query is $query"); -# $status_id = "4"; #db.conn.GetRow($query) - my $sth = $dbh->prepare($query); - $sth->execute($status); + my $sth = run_query($query,$status); $status_id = $sth->fetchrow_hashref->{'statusid'}; }; ($@) and print "ERROR Could not get status id: $DBI::errstr\n"; @@ -299,11 +286,18 @@ sub set_job_status { eval { my $query = 'INSERT INTO job_history (jobid,clientid,statusid,eventmsg) VALUES (?,?,?,?)'; - debug("Query is $query"); - my $sth = $dbh->prepare($query); - $sth->execute($jobid,$id_of_client,$status_id,$eventmsg); + run_query($query,$jobid,$id_of_client,$status_id,$eventmsg); }; ($@) and print "ERROR Could not insert into job_history: $DBI::errstr\n"; + + + #If we're marking the completetion or failure of a job, we have more work to do here. + if ($status eq 'Failed') { + mark_job_as_failed($jobid,$id_of_client); + } elsif ($status eq 'Completed') { + mark_job_as_completed($jobid,$id_of_client); + } + return 1; } @@ -318,6 +312,15 @@ sub parse_command { return @parts; } +sub run_query { + my ($query, @params) = @_; + debug("Query is $query"); + my $sth = $dbh->prepare($query); + $sth->execute(@params); + return $sth; +} + + sub expand_jobs { #Searches for the group jobs that the client must be into and does the expansion. my @groups = get_client_groups(); @@ -333,39 +336,60 @@ AND (job_conditions.deploy_time < now()) AND ((job_conditions.expiration_time > now()) OR (job_conditions.expiration_time IS NULL)) AND ((job_conditions.last_run_date < job_conditions.deploy_time) OR (job_conditions.last_run_date IS NULL)) EndOfQuery2 - debug("Query is $query"); - my $sth = $dbh->prepare($query); - $sth->execute($groupid); - $dbh->do('LOCK TABLES `jobs_clients` WRITE, `job_conditions` WRITE, `job_history` WRITE, `jobs_status` WRITE'); + my $sth = run_query($query,$groupid); + run_query('LOCK TABLES `jobs_clients` WRITE, `job_conditions` WRITE, `job_history` WRITE, `jobs_status` WRITE, `jobs` WRITE'); + #FIXME need to lock jobs_clients for READ as well!!! while( my $jobref = $sth->fetchrow_hashref() ) { my $jobid = $jobref->{'jobid'}; foreach my $member (@members) { $query = 'INSERT INTO jobs_clients (jobid, clientid) VALUES (?,?)'; - debug("Query is $query"); - my $sth2 = $dbh->prepare($query); - $sth2->execute($jobid,$member); + my $sth2 = run_query($query,$jobid,$member); set_job_status($jobid,$member,'Pending', 'Job expanded.') or print "ERROR could not add expanded jobs to job_history.\n"; } $query = 'UPDATE `job_conditions` SET last_run_date = now() WHERE jobid = ?'; - debug("Query is $query"); - my $sth3 = $dbh->prepare($query); - $sth3->execute($jobid); + run_query($query,$jobid); + + $query = 'UPDATE `jobs` SET pending=pending+? WHERE jobid = ?'; + run_query($query,$#members,$jobid); #This works because you want one less b/c of removing the group. # One last query to remove the row from jobs_clients so someone else doesn't expand it. $query = 'DELETE FROM `jobs_clients` WHERE groupid=? AND jobid=?'; - debug("Query is $query"); - my $sth4 = $dbh->prepare($query); - $sth4->execute($groupid,$jobid); + run_query($query,$groupid,$jobid); + } - $dbh->do('UNLOCK TABLES'); + run_query('UNLOCK TABLES'); }; ($@) and print "ERROR Could not expand jobs: $@ $DBI::errstr\n"; return undef; } } +sub mark_job_as_failed { + my ($jobid,$id_of_client) = @_; +} + +sub mark_job_as_completed { + my ($jobid,$id_of_client) = @_; + my ($query,$sth); + debug("Marking $jobid as completed for client $id_of_client"); + #If we succeeded, we need to check this jobid to see if it is a recurring job, and then set the next_run if necessary. + #This requries looking at the pending count for the job as well as the run_schedule. + + #First off, update the pending count now that we've finished. + eval { + $query = 'UPDATE jobs SET pending=pending-1 WHERE jobid=?'; + debug("Query is $query"); + }; + ($@) and print "ERROR Could not update pending count: $@ $DBI::errstr\n"; +} + +sub process_jobfile { + my $filename = shift; + +} + ######################################################### # PHPGACL FUNCTIONS ######################################################### -- cgit v1.2.3-65-gdbad From 68fe47ae92bba9bbf3cf35b79134c65317c1b258 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 05:24:56 +0000 Subject: only import open2 from IPC::Open2 svn path=/branches/new-fu/; revision=328 --- client/Scire.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/Scire.pm b/client/Scire.pm index dca8da4..8fd5227 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -35,7 +35,7 @@ sub set_filename { package Scire::Communicator; -use IPC::Open2; +use IPC::Open2 (open2); sub new { my $proto = shift; -- cgit v1.2.3-65-gdbad From e4c4bd5bf3d74740f713fabb86cb4e92aa23d781 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 05:28:11 +0000 Subject: change set_filename() to load_jobfile() svn path=/branches/new-fu/; revision=329 --- client/Scire.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/Scire.pm b/client/Scire.pm index 8fd5227..19c6119 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -7,12 +7,12 @@ sub new { my $self = {}; bless ($self, $class); if(defined $filename) { - $self->set_filename($filename); + $self->load_jobfile($filename); } return $self; } -sub set_filename { +sub load_jobfile { my $self = shift; my $filename = shift; $self->{filename} = $filename; -- cgit v1.2.3-65-gdbad From 55da87d03dc126a1fda61d5f271aba3bea8165a5 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 5 Jan 2008 06:26:21 +0000 Subject: move code that talks to server into talk_to_server(), which forks before doing its thing in order to drop privileges svn path=/branches/new-fu/; revision=330 --- client/Scire.pm | 6 ++++++ client/scireclient.pl | 60 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/client/Scire.pm b/client/Scire.pm index 19c6119..2275a87 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -96,6 +96,12 @@ sub create_connection { $self->{connection_pid} = open2($self->{SERVER_STDOUT}, $self->{SERVER_STDIN}, $self->{connection_command}); } +sub close_connection { + my $self = shift; + close $self->{SERVER_STDIN}; + close $self->{SERVER_STDOUT}; +} + sub build_connection_command { my $self = shift; # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" diff --git a/client/scireclient.pl b/client/scireclient.pl index 05d2340..f630f7f 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -10,6 +10,7 @@ use Getopt::Long; use Data::Dumper; use File::Path; use Sys::Hostname; +use POSIX qw/WEXITSTATUS setuid/; my $ETC_DIR = "/etc/scire"; my $SCIRE_CONFIG_FILE = "${ETC_DIR}/scire.conf"; @@ -25,34 +26,53 @@ sub run_main { check_job_dir(); - #ok folks so here's how this thang goes down. - #1. Connect. - $comm = Scire::Communicator->new( host => $conf{host}, user => $conf{user}, port => $conf{port} ); - $comm->create_connection(); - - #2. Register with the DB. (only it knows if you're allowed to be active) - # If we do not have a defined key file, we assume this is the first run of this client - # so we register them instead of trying to identify. - if(defined($conf{key_file}) and (-f $conf{key_file})) { - if(!identify_client()) { - exit(1); - } + my $exitcode = talk_to_server(); +} + +sub talk_to_server { + # This functions forks a new process just for the purpose of dropping privileges. + my $pid = fork(); + if($pid) { + debug("Waiting for PID ${pid} to finish"); + waitpid($pid, 0); + my $exitcode = WEXITSTATUS($?); + debug("PID ${pid} has finished with status ${exitcode}"); + return $exitcode; } else { - register_client(); + # We'll need to add a call to setuid() here at some point + #ok folks so here's how this thang goes down. + #1. Connect. + $comm = Scire::Communicator->new( host => $conf{host}, user => $conf{user}, port => $conf{port} ); + $comm->create_connection(); + + #2. Register with the DB. (only it knows if you're allowed to be active) + # If we do not have a defined key file, we assume this is the first run of this client + # so we register them instead of trying to identify. + if(defined($conf{key_file}) and (-f $conf{key_file})) { + if(!identify_client()) { + exit(1); + } + } else { + register_client(); + exit(0); + } + + #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. + my @existing_jobs = scan_jobs_dir(); + #4. Fetch the jobs list + get_jobs(); + #5. ??? + #6. Profit! + + $comm->close_connection(); exit(0); } - - #3. Scan the jobs directory. If there are done/failed jobs, report them. Note jobs in running or queue. - my @existing_jobs = scan_jobs_dir(); - #4. Fetch the jobs list - get_jobs(); - #5. ??? - #6. Profit! } sub parse_command_line { GetOptions( 'debug|d' => \$conf{debug}, + 'daemon|D' => \$conf{daemon}, 'dry-run' => \$conf{dry_run}, 'help|h' => \$conf{help}, 'config|c=s' => \$conf{config}, -- cgit v1.2.3-65-gdbad From 91cae4e0987b44313ebb091bb1cd376130c5a4be Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 07:05:55 +0000 Subject: condense check_job_dir() with a loop svn path=/branches/new-fu/; revision=331 --- client/scireclient.pl | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index f630f7f..2415373 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -40,6 +40,7 @@ sub talk_to_server { return $exitcode; } else { # We'll need to add a call to setuid() here at some point + #ok folks so here's how this thang goes down. #1. Connect. $comm = Scire::Communicator->new( host => $conf{host}, user => $conf{user}, port => $conf{port} ); @@ -96,25 +97,13 @@ sub parse_command_line { } sub check_job_dir { - if (! -d $conf{job_dir}) { - print "WARNING! $conf{job_dir} does not exist...creating\n"; - mkpath( $conf{job_dir}, {verbose => 1, mode => 0660}) - or die("Couldn't make $conf{job_dir} w/ perms 0660: $!"); - } - if (! -d "$conf{job_dir}/queue") { - print "WARNING! $conf{job_dir}/queue does not exist...creating\n"; - mkpath( "$conf{job_dir}/queue", {verbose => 1, mode => 0660}) - or die("Couldn't make $conf{job_dir}/queue w/ perms 0660: $!"); - } - if (! -d "$conf{job_dir}/done") { - print "WARNING! $conf{job_dir}/done does not exist...creating\n"; - mkpath( "$conf{job_dir}/done", {verbose => 1, mode => 0660}) - or die("Couldn't make $conf{job_dir}/done w/ perms 0660: $!"); - } - if (! -d "$conf{job_dir}/failed") { - print "WARNING! $conf{job_dir}/failed does not exist...creating\n"; - mkpath( "$conf{job_dir}/failed", {verbose => 1, mode => 0660}) - or die("Couldn't make $conf{job_dir}/failed w/ perms 0660: $!"); + my @checkdirs = ($conf{job_dir}, "$conf{job_dir}/queue", "$conf{job_dir}/done", "$conf{job_dir}/failed"); + for my $dir (@checkdirs) { + if (! -d $dir) { + print "WARNING! ${dir} does not exist...creating\n"; + mkpath( $dir, {verbose => 1, mode => 0660}) + or die("Couldn't make ${dir} w/ perms 0660: $!"); + } } } -- cgit v1.2.3-65-gdbad From 6eec6a582aaad4828710e46162e9e5fc933e3a3e Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 07:35:46 +0000 Subject: add stub for run() svn path=/branches/new-fu/; revision=332 --- client/Scire.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/Scire.pm b/client/Scire.pm index 2275a87..455c8b1 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -33,6 +33,11 @@ sub load_jobfile { } } +sub run { + my ($self, $outfile, $errfile) = @_; + +} + package Scire::Communicator; use IPC::Open2 (open2); -- cgit v1.2.3-65-gdbad From 21186444cd1411fbba30bbf274a3c8aaf6b060da Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 08:09:17 +0000 Subject: fill out run() a bit more svn path=/branches/new-fu/; revision=333 --- client/Scire.pm | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/client/Scire.pm b/client/Scire.pm index 455c8b1..deedcb6 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -1,5 +1,8 @@ package Scire::Job; +use POSIX qw/WEXITSTATUS WIFEXITED waitpid/; +use IPC::Open3 (open3); + sub new { my $proto = shift; my $class = ref($proto) || $proto; @@ -33,9 +36,50 @@ sub load_jobfile { } } +sub set_stdout_file { + my ($self, $outfile) = @_; + if(defined $outfile && $outfile) { + $self->{stdout_filename} = $outfile; + } +} + +sub set_stderr_file { + my ($self, $errfile) = @_; + if(defined $errfile && $errfile) { + $self->{stderr_filename} = $errfile; + } +} + sub run { - my ($self, $outfile, $errfile) = @_; + my $self = shift; + # XXX: write $self->{script_data} out to a file and mark it +x. We'll need + # to find a good location for this, since it can't be noexec. Perhaps the + # queue dir in the job directory will do, or maybe it will be configurable + + my $pid = fork(); + if($fork) { + # XXX: eventually, we'll move the waitpid() call to another function + # called something like is_running() and use WNOHANG instead of blocking + waitpid($pid, 0); + my $status = $?; + my $exitcode = -1; + if(WIFEXITED($status)) { + my $exitcode = WEXITSTATUS($status); + } + return $exitcode; + } else { + # XXX: we'll use setuid to drop privileges here + if(defined $self->{stdout_filename}) { + open STDOUT, '>', $self->{stdout_filename}; + } + if(defined $self->{stderr_filename}) { + open STDERR, '>', $self->{stderr_filename}; + } + # XXX: exec() to run our command. our STDOUT and STDERR have been + # redirected to the files specified, and the exit code is returned + # to the main process when we're done executing + } } package Scire::Communicator; -- cgit v1.2.3-65-gdbad From e5613af09e3bccfa0b8d4eb8397f0f816d0ed20b Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 6 Jan 2008 16:27:36 +0000 Subject: break out the Communicator code. svn path=/branches/new-fu/; revision=334 --- client/Scire.pm | 88 ------------------------------------------- client/Scire/Communicator.pm | 89 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 88 deletions(-) create mode 100644 client/Scire/Communicator.pm diff --git a/client/Scire.pm b/client/Scire.pm index deedcb6..7319529 100644 --- a/client/Scire.pm +++ b/client/Scire.pm @@ -82,94 +82,6 @@ sub run { } } -package Scire::Communicator; - -use IPC::Open2 (open2); - -sub new { - my $proto = shift; - my $class = ref($proto) || $proto; - my $self = { - port => 22, - user => scire, - server_script => "/usr/bin/scireserver.pl", - SERVER_STDOUT => undef, - SERVER_STDIN => undef, - @_ - }; - bless ($self, $class); - $self->build_connection_command(); - return $self; -} - -sub send_command { - my $self = shift; - my $cmd = shift; - my @args = @_; - my $tosend = "${cmd}"; - - for my $arg (@args) { - if($arg =~ /^[0-9]+$/) { - $tosend .= " ${arg}"; - } else { - $arg =~ s/"/\\"/g; - $tosend .= " \"${arg}\""; - } - } - $tosend .= "\n"; - - my ($tmpin, $tmpout) = ($self->{SERVER_STDIN}, $self->{SERVER_STDOUT}); - print $tmpin $tosend; - #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! - #if the server doesn't give you a newline this just hangs! - my $response = <$tmpout>; - return $self->parse_response($response); -} - -sub parse_response { - my $self = shift; - my $response = shift; - $response =~ /^(OK|ERROR)(?: (.+?))?\s*$/; - my ($status, $message) = ($1, $2); - return ($status, $message); -} - -sub create_connection { - my $self = shift; - # XXX: How do we capture this error? $pid has a valid value even if the - # process fails to run, since it just returns the PID of the forked perl - # process. I tried adding 'or die' after it, but it didn't help since it - # doesn't fail in the main process. When it fails, it outputs an error - # to STDERR: - # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 - $self->{connection_pid} = open2($self->{SERVER_STDOUT}, $self->{SERVER_STDIN}, $self->{connection_command}); -} - -sub close_connection { - my $self = shift; - close $self->{SERVER_STDIN}; - close $self->{SERVER_STDOUT}; -} - -sub build_connection_command { - my $self = shift; - # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" - my $connection_command = "ssh "; - $connection_command .= "-o BatchMode yes "; - $connection_command .= "-o SendEnv 'SCIRE_*' "; - $connection_command .= "-o ServerAliveInterval 15 -o ServerAliveCountMax 4 "; - if(defined($self->{port})) { - $connection_command .= "-o Port=$conf{port} "; - } - $connection_command .= "$self->{user}\@$self->{host} $self->{server_script}"; - - if (-d ".svn") { - # Overwrite $connection_command in the case of a dev environment for now - $connection_command = "../server/scireserver.pl"; - } - $self->{connection_command} = $connection_command; -} 1; - diff --git a/client/Scire/Communicator.pm b/client/Scire/Communicator.pm new file mode 100644 index 0000000..1a6b982 --- /dev/null +++ b/client/Scire/Communicator.pm @@ -0,0 +1,89 @@ +package Scire::Communicator; + +use IPC::Open2 (open2); + +sub new { + my $proto = shift; + my $class = ref($proto) || $proto; + my $self = { + port => 22, + user => scire, + server_script => "/usr/bin/scireserver.pl", + SERVER_STDOUT => undef, + SERVER_STDIN => undef, + @_ + }; + bless ($self, $class); + $self->build_connection_command(); + return $self; +} + +sub send_command { + my $self = shift; + my $cmd = shift; + my @args = @_; + my $tosend = "${cmd}"; + + for my $arg (@args) { + if($arg =~ /^[0-9]+$/) { + $tosend .= " ${arg}"; + } else { + $arg =~ s/"/\\"/g; + $tosend .= " \"${arg}\""; + } + } + $tosend .= "\n"; + + my ($tmpin, $tmpout) = ($self->{SERVER_STDIN}, $self->{SERVER_STDOUT}); + print $tmpin $tosend; + #FIXME WE NEED A TIMEOUT HERE OF SOME SORT!! + #if the server doesn't give you a newline this just hangs! + my $response = <$tmpout>; + return $self->parse_response($response); +} + +sub parse_response { + my $self = shift; + my $response = shift; + $response =~ /^(OK|ERROR)(?: (.+?))?\s*$/; + my ($status, $message) = ($1, $2); + return ($status, $message); +} + +sub create_connection { + my $self = shift; + # XXX: How do we capture this error? $pid has a valid value even if the + # process fails to run, since it just returns the PID of the forked perl + # process. I tried adding 'or die' after it, but it didn't help since it + # doesn't fail in the main process. When it fails, it outputs an error + # to STDERR: + # open2: exec of ../server/scireserver.pl failed at ./scireclient.pl line 116 + $self->{connection_pid} = open2($self->{SERVER_STDOUT}, $self->{SERVER_STDIN}, $self->{connection_command}); +} + +sub close_connection { + my $self = shift; + close $self->{SERVER_STDIN}; + close $self->{SERVER_STDOUT}; +} + +sub build_connection_command { + my $self = shift; + # This will eventually be something like "ssh scire@${scireserver} /usr/bin/scireserver.pl" + my $connection_command = "ssh "; + $connection_command .= "-o BatchMode yes "; + $connection_command .= "-o SendEnv 'SCIRE_*' "; + $connection_command .= "-o ServerAliveInterval 15 -o ServerAliveCountMax 4 "; + if(defined($self->{port})) { + $connection_command .= "-o Port=$conf{port} "; + } + $connection_command .= "$self->{user}\@$self->{host} $self->{server_script}"; + + if (-d ".svn") { + # Overwrite $connection_command in the case of a dev environment for now + $connection_command = "../server/scireserver.pl"; + } + $self->{connection_command} = $connection_command; +} + +1; -- cgit v1.2.3-65-gdbad From 8cddf3d973323e0965ab0a3e190f131888a37e5c Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 17:31:40 +0000 Subject: move Scire.pm to Scire/Job.pm and fix scireclient.pm to use the new paths svn path=/branches/new-fu/; revision=335 --- client/Scire.pm | 87 --------------------------------------------------- client/Scire/Job.pm | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++ client/scireclient.pl | 3 +- 3 files changed, 89 insertions(+), 88 deletions(-) delete mode 100644 client/Scire.pm create mode 100644 client/Scire/Job.pm diff --git a/client/Scire.pm b/client/Scire.pm deleted file mode 100644 index 7319529..0000000 --- a/client/Scire.pm +++ /dev/null @@ -1,87 +0,0 @@ -package Scire::Job; - -use POSIX qw/WEXITSTATUS WIFEXITED waitpid/; -use IPC::Open3 (open3); - -sub new { - my $proto = shift; - my $class = ref($proto) || $proto; - my $filename = shift; - my $self = {}; - bless ($self, $class); - if(defined $filename) { - $self->load_jobfile($filename); - } - return $self; -} - -sub load_jobfile { - my $self = shift; - my $filename = shift; - $self->{filename} = $filename; - my $jobcontents; - my $jobdata; - open JOB, "< ${filename}" or die "Can't open file ${filename}"; - $jobcontents = join("", ); - close JOB; - $jobdata = eval($jobcontents); - ($@) and print "ERROR: Could not parse job file ${filename}!\n"; - if(defined $jobdata->{script}) { - for(keys %{$jobdata->{script}}) { - $self->{$_} = $jobdata->{script}->{$_}; - } - } - for(keys %{$jobdata}) { - $self->{$_} = $jobdata->{$_} unless($_ eq "script"); - } -} - -sub set_stdout_file { - my ($self, $outfile) = @_; - if(defined $outfile && $outfile) { - $self->{stdout_filename} = $outfile; - } -} - -sub set_stderr_file { - my ($self, $errfile) = @_; - if(defined $errfile && $errfile) { - $self->{stderr_filename} = $errfile; - } -} - -sub run { - my $self = shift; - - # XXX: write $self->{script_data} out to a file and mark it +x. We'll need - # to find a good location for this, since it can't be noexec. Perhaps the - # queue dir in the job directory will do, or maybe it will be configurable - - my $pid = fork(); - if($fork) { - # XXX: eventually, we'll move the waitpid() call to another function - # called something like is_running() and use WNOHANG instead of blocking - waitpid($pid, 0); - my $status = $?; - my $exitcode = -1; - if(WIFEXITED($status)) { - my $exitcode = WEXITSTATUS($status); - } - return $exitcode; - } else { - # XXX: we'll use setuid to drop privileges here - if(defined $self->{stdout_filename}) { - open STDOUT, '>', $self->{stdout_filename}; - } - if(defined $self->{stderr_filename}) { - open STDERR, '>', $self->{stderr_filename}; - } - # XXX: exec() to run our command. our STDOUT and STDERR have been - # redirected to the files specified, and the exit code is returned - # to the main process when we're done executing - } -} - - - -1; diff --git a/client/Scire/Job.pm b/client/Scire/Job.pm new file mode 100644 index 0000000..7319529 --- /dev/null +++ b/client/Scire/Job.pm @@ -0,0 +1,87 @@ +package Scire::Job; + +use POSIX qw/WEXITSTATUS WIFEXITED waitpid/; +use IPC::Open3 (open3); + +sub new { + my $proto = shift; + my $class = ref($proto) || $proto; + my $filename = shift; + my $self = {}; + bless ($self, $class); + if(defined $filename) { + $self->load_jobfile($filename); + } + return $self; +} + +sub load_jobfile { + my $self = shift; + my $filename = shift; + $self->{filename} = $filename; + my $jobcontents; + my $jobdata; + open JOB, "< ${filename}" or die "Can't open file ${filename}"; + $jobcontents = join("", ); + close JOB; + $jobdata = eval($jobcontents); + ($@) and print "ERROR: Could not parse job file ${filename}!\n"; + if(defined $jobdata->{script}) { + for(keys %{$jobdata->{script}}) { + $self->{$_} = $jobdata->{script}->{$_}; + } + } + for(keys %{$jobdata}) { + $self->{$_} = $jobdata->{$_} unless($_ eq "script"); + } +} + +sub set_stdout_file { + my ($self, $outfile) = @_; + if(defined $outfile && $outfile) { + $self->{stdout_filename} = $outfile; + } +} + +sub set_stderr_file { + my ($self, $errfile) = @_; + if(defined $errfile && $errfile) { + $self->{stderr_filename} = $errfile; + } +} + +sub run { + my $self = shift; + + # XXX: write $self->{script_data} out to a file and mark it +x. We'll need + # to find a good location for this, since it can't be noexec. Perhaps the + # queue dir in the job directory will do, or maybe it will be configurable + + my $pid = fork(); + if($fork) { + # XXX: eventually, we'll move the waitpid() call to another function + # called something like is_running() and use WNOHANG instead of blocking + waitpid($pid, 0); + my $status = $?; + my $exitcode = -1; + if(WIFEXITED($status)) { + my $exitcode = WEXITSTATUS($status); + } + return $exitcode; + } else { + # XXX: we'll use setuid to drop privileges here + if(defined $self->{stdout_filename}) { + open STDOUT, '>', $self->{stdout_filename}; + } + if(defined $self->{stderr_filename}) { + open STDERR, '>', $self->{stderr_filename}; + } + # XXX: exec() to run our command. our STDOUT and STDERR have been + # redirected to the files specified, and the exit code is returned + # to the main process when we're done executing + } +} + + + +1; diff --git a/client/scireclient.pl b/client/scireclient.pl index 2415373..91bd1c5 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -5,7 +5,8 @@ use strict; use warnings; -use Scire; +use Scire::Job; +use Scire::Communicator; use Getopt::Long; use Data::Dumper; use File::Path; -- cgit v1.2.3-65-gdbad From d8dd84b9e57bb003476433ec5a8d7e8df8c8c319 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 17:32:33 +0000 Subject: remove unneeded module svn path=/branches/new-fu/; revision=336 --- client/Scire/Job.pm | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/Scire/Job.pm b/client/Scire/Job.pm index 7319529..25d9ce8 100644 --- a/client/Scire/Job.pm +++ b/client/Scire/Job.pm @@ -1,7 +1,6 @@ package Scire::Job; use POSIX qw/WEXITSTATUS WIFEXITED waitpid/; -use IPC::Open3 (open3); sub new { my $proto = shift; @@ -82,6 +81,4 @@ sub run { } } - - 1; -- cgit v1.2.3-65-gdbad From c0d8efc8a831c4f8a391731040d04b42b28c2dac Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 20:02:24 +0000 Subject: add in exec() call svn path=/branches/new-fu/; revision=337 --- client/Scire/Job.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/Scire/Job.pm b/client/Scire/Job.pm index 25d9ce8..9216116 100644 --- a/client/Scire/Job.pm +++ b/client/Scire/Job.pm @@ -77,7 +77,9 @@ sub run { } # XXX: exec() to run our command. our STDOUT and STDERR have been # redirected to the files specified, and the exit code is returned - # to the main process when we're done executing + # to the main process when we're done executing. This will be changed + # to the path of the script we've written to disk once that code is in + exec '/bin/sh', '-c', 'echo foo; echo bar >&2'; } } -- cgit v1.2.3-65-gdbad From ca20aa4572c3614393af1a889ffd731caf0764e0 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 22:01:27 +0000 Subject: check proper variable after forking add code to setuid() before job execution svn path=/branches/new-fu/; revision=339 --- client/Scire/Job.pm | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/client/Scire/Job.pm b/client/Scire/Job.pm index 9216116..913e3de 100644 --- a/client/Scire/Job.pm +++ b/client/Scire/Job.pm @@ -57,7 +57,7 @@ sub run { # queue dir in the job directory will do, or maybe it will be configurable my $pid = fork(); - if($fork) { + if($pid) { # XXX: eventually, we'll move the waitpid() call to another function # called something like is_running() and use WNOHANG instead of blocking waitpid($pid, 0); @@ -68,13 +68,26 @@ sub run { } return $exitcode; } else { - # XXX: we'll use setuid to drop privileges here + # We redirect STDOUT and STDERR first since the new user may not have + # write access to the file locations if(defined $self->{stdout_filename}) { open STDOUT, '>', $self->{stdout_filename}; } if(defined $self->{stderr_filename}) { open STDERR, '>', $self->{stderr_filename}; } + # XXX: we might want to check capabilities here instead of UID, but I + # have no idea how to do that + if($< == 0) { + # XXX: we'll use setuid to drop privileges here + my $user = getpwnam($self->{run_as}); + if(defined $user) { + setuid($user[2]); + } else { + # XXX: the specified user does not exist. we should really do + # something here + } + } # XXX: exec() to run our command. our STDOUT and STDERR have been # redirected to the files specified, and the exit code is returned # to the main process when we're done executing. This will be changed -- cgit v1.2.3-65-gdbad From 3b309e0a28e1b80d61db6073289f2ab90df62c56 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sun, 6 Jan 2008 22:10:00 +0000 Subject: exit with -2 if run_as user can't be found always try to get exitstatus svn path=/branches/new-fu/; revision=340 --- client/Scire/Job.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/Scire/Job.pm b/client/Scire/Job.pm index 913e3de..0b016dc 100644 --- a/client/Scire/Job.pm +++ b/client/Scire/Job.pm @@ -62,10 +62,10 @@ sub run { # called something like is_running() and use WNOHANG instead of blocking waitpid($pid, 0); my $status = $?; - my $exitcode = -1; - if(WIFEXITED($status)) { +# my $exitcode = -1; +# if(WIFEXITED($status)) { my $exitcode = WEXITSTATUS($status); - } +# } return $exitcode; } else { # We redirect STDOUT and STDERR first since the new user may not have @@ -86,6 +86,7 @@ sub run { } else { # XXX: the specified user does not exist. we should really do # something here + exit(-2); } } # XXX: exec() to run our command. our STDOUT and STDERR have been -- cgit v1.2.3-65-gdbad From c8e2a70101d80668aa3e3a8141dc3301bfa0da93 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 7 Jan 2008 01:09:58 +0000 Subject: write out script_data to file determine UID/GID of run_as user beforehand so we can change the owner of the script we just wrote exec the script we wrote out instead of dummy commands add set_script_file() function svn path=/branches/new-fu/; revision=341 --- client/Scire/Job.pm | 50 +++++++++++++++++++++++++++++++++----------------- client/scireclient.pl | 2 +- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/client/Scire/Job.pm b/client/Scire/Job.pm index 0b016dc..27a5aa7 100644 --- a/client/Scire/Job.pm +++ b/client/Scire/Job.pm @@ -1,6 +1,6 @@ package Scire::Job; -use POSIX qw/WEXITSTATUS WIFEXITED waitpid/; +use POSIX qw/WEXITSTATUS WIFEXITED waitpid setuid setgid/; sub new { my $proto = shift; @@ -35,6 +35,13 @@ sub load_jobfile { } } +sub set_script_file { + my ($self, $scriptfile) = @_; + if(defined $scriptfile and $scriptfile) { + $self->{script_filename} = $scriptfile; + } +} + sub set_stdout_file { my ($self, $outfile) = @_; if(defined $outfile && $outfile) { @@ -52,9 +59,27 @@ sub set_stderr_file { sub run { my $self = shift; - # XXX: write $self->{script_data} out to a file and mark it +x. We'll need - # to find a good location for this, since it can't be noexec. Perhaps the - # queue dir in the job directory will do, or maybe it will be configurable + # XXX: we might want to check capabilities here instead of UID, but I + # have no idea how to do that + my ($run_as_uid, $run_as_gid) = (0, 0); + if($< == 0) { + # XXX: we'll use setuid to drop privileges here + my @user = getpwnam($self->{run_as}); + if(defined @user) { + $run_as_uid = $user[2]; + $run_as_gid = $user[3]; + } else { + return -2; + } + } + + open SCRIPT, ">", $self->{script_filename}; + print SCRIPT $self->{script_data}; + close SCRIPT; + if($run_as_uid) { + chown $run_as_uid, $run_as_gid, $self->{script_filename}; + } + chmod 0500, $self->{script_filename}; my $pid = fork(); if($pid) { @@ -76,24 +101,15 @@ sub run { if(defined $self->{stderr_filename}) { open STDERR, '>', $self->{stderr_filename}; } - # XXX: we might want to check capabilities here instead of UID, but I - # have no idea how to do that - if($< == 0) { - # XXX: we'll use setuid to drop privileges here - my $user = getpwnam($self->{run_as}); - if(defined $user) { - setuid($user[2]); - } else { - # XXX: the specified user does not exist. we should really do - # something here - exit(-2); - } + if($run_as_uid) { + setuid($run_as_uid); + setgid($run_as_gid); } # XXX: exec() to run our command. our STDOUT and STDERR have been # redirected to the files specified, and the exit code is returned # to the main process when we're done executing. This will be changed # to the path of the script we've written to disk once that code is in - exec '/bin/sh', '-c', 'echo foo; echo bar >&2'; + exec '/bin/sh', '-c', $self->{script_filename}; } } diff --git a/client/scireclient.pl b/client/scireclient.pl index 91bd1c5..304051e 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -98,7 +98,7 @@ sub parse_command_line { } sub check_job_dir { - my @checkdirs = ($conf{job_dir}, "$conf{job_dir}/queue", "$conf{job_dir}/done", "$conf{job_dir}/failed"); + my @checkdirs = ($conf{job_dir}, "$conf{job_dir}/queue", "$conf{job_dir}/done", "$conf{job_dir}/failed", "$conf{job_dir}/run"); for my $dir (@checkdirs) { if (! -d $dir) { print "WARNING! ${dir} does not exist...creating\n"; -- cgit v1.2.3-65-gdbad From a64e00957fbdb68119d9910438b291844005a429 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 7 Jan 2008 01:23:06 +0000 Subject: handle a non-zero exitcode from the communicator svn path=/branches/new-fu/; revision=342 --- client/scireclient.pl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/scireclient.pl b/client/scireclient.pl index 304051e..b4b0227 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -28,6 +28,15 @@ sub run_main { check_job_dir(); my $exitcode = talk_to_server(); + if($exitcode != 0) { + if($conf{daemon}) { + # We'll schedule another pass here later + } else { + debug("We couldn't communicate with the server...exiting!"); + exit(1); + } + } + } sub talk_to_server { -- cgit v1.2.3-65-gdbad From 814df59ab6038e7c36b1fb2de76cabf9a23cb7c8 Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Mon, 7 Jan 2008 03:38:31 +0000 Subject: attempt to add the recurring job rescheduling into the server. not tested. svn path=/branches/new-fu/; revision=343 --- server/scireserver.pl | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index 68d6fe9..f577f1e 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -8,6 +8,8 @@ use DBI; use Data::Dumper; use Digest::MD5 qw(md5 md5_hex ); use File::Path; +use Schedule::Cron::Events; + $| = 1; $Data::Dumper::Purity = 1; @@ -368,6 +370,13 @@ EndOfQuery2 sub mark_job_as_failed { my ($jobid,$id_of_client) = @_; + + #First off, update the pending count and failed count with the result. + eval { + my $query = 'UPDATE jobs SET pending=pending-1, failed=failed+1 WHERE jobid=?'; + run_query($query); + }; + ($@) and print "ERROR Could not update pending count: $@ $DBI::errstr\n"; } sub mark_job_as_completed { @@ -380,9 +389,35 @@ sub mark_job_as_completed { #First off, update the pending count now that we've finished. eval { $query = 'UPDATE jobs SET pending=pending-1 WHERE jobid=?'; - debug("Query is $query"); + run_query($query,$jobid); }; ($@) and print "ERROR Could not update pending count: $@ $DBI::errstr\n"; + + #Now get the pending count and run_schedule. + eval { + $query = 'SELECT pending,run_schedule,expiration_time,deploy_time FROM jobs WHERE jobid=?'; + my $sth = run_query($query,$jobid); + my $rowref = $sth->fetchrow_hashref(); + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); + my $datetime = sprintf "%4d-%02d-%02d %02d:%02d:%02d\n",$year+1900,$mon+1,$mday,$hour,$min,$sec; + if ($rowref->{'run_schedule'} and ($rowref->{'pending'} == 0) and ( $rowref->{'expiration_time'} > $datetime)) { + #Determine the next run time. + my $cron = new Schedule::Cron::Events( $rowref->{'run_schedule'}, Date => [ ( localtime(time()) )[0..5] ] ); + my ($sec, $min, $hour, $day, $month, $year) = $cron->nextEvent; + printf("Event will start next at %2d:%02d:%02d on %d %s, %d\n", $hour, $min, $sec, $day, $month, ($year+1900)); + + #Get the groups and clients from the recurring_jobs_clients table. + $query = 'SELECT clientid,groupid FROM recurring_jobs_clients WHERE jobid=?'; + my $sth2 = run_query($query,$jobid); + while( my $recrow_ref = $sth2->fetchrow_hashref() ) { + $query = 'INSERT INTO jobs_clients (jobid,clientid,groupid) VALUES (?,?,?)'; + run_query($query,$jobid,$recrow_ref->{'clientid'},$recrow_ref->{'groupid'}); + } + } + + }; + ($@) and print "ERROR Could not get run_schedule and pending count: $@ $DBI::errstr\n"; + } sub process_jobfile { -- cgit v1.2.3-65-gdbad From 0e65917503a37e661826a3eaa521a22b5d31c38a Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Mon, 7 Jan 2008 03:59:27 +0000 Subject: basic support for sending both stdout and stderr svn path=/branches/new-fu/; revision=344 --- server/scireserver.pl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index f577f1e..fa5de61 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -94,12 +94,12 @@ while(<>) { set_job_status($jobid,$client_id,$status) and print "OK\n"; } elsif ($command eq "RETURN_JOBFILE") { my $jobid = $args[0]; - my $filename = "$conf{job_dir}/$client_id/result/$jobid.result"; - print "OK ${filename}\n"; + my @filenames = ("$conf{job_dir}/$client_id/result/$jobid.stdout", "$conf{job_dir}/$client_id/result/$jobid.stderr"); + print "OK " . join(" ", @filenames) . "\n"; } elsif ($command eq "JOBFILE_SENT") { - my $filename = $args[0]; - print "OK\n" and process_jobfile($filename); - + my @filenames = @args; + process_jobfile($_) foreach(@jobfiles); + print "OK\n" } else { print "ERROR The command $command is unknown. Please try again.\n"; } -- cgit v1.2.3-65-gdbad From cc2c81b10e0bbb28ab1f3e6362de7ad316f0f748 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Tue, 8 Jan 2008 02:23:26 +0000 Subject: test script for Scire::Job svn path=/branches/new-fu/; revision=345 --- client/test.pl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 client/test.pl diff --git a/client/test.pl b/client/test.pl new file mode 100644 index 0000000..c198964 --- /dev/null +++ b/client/test.pl @@ -0,0 +1,26 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +$| = 1; + +use Scire::Job; +use Scire::Communicator; + +my $job = Scire::Job->new(); +$job->load_jobfile("/tmp/scirejobs/job1.job"); +#print $job->{script_data} . "\n"; +$job->set_stdout_file("/tmp/scire_stdout.txt"); +$job->set_stderr_file("/tmp/scire_stderr.txt"); +$job->set_script_file("/tmp/scirejobs/runjob.sh"); +my $exitcode = $job->run(); +print "Job complete with exit code ${exitcode}\n"; + +exit; + +my $comm = Scire::Communicator->new( host => "localhost" ); +$comm->create_connection(); +$comm->close_connection(); +#my ($status, $message) = $comm->send_command("QUIT"); +#print "$status\n"; -- cgit v1.2.3-65-gdbad From be3472e11b418f24ab4f14021c4e2e630af6fd56 Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 12 Jan 2008 03:39:59 +0000 Subject: use proper form for send_command() svn path=/branches/new-fu/; revision=349 --- client/scireclient.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index b4b0227..c054e17 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -204,7 +204,7 @@ sub scan_jobs_dir { foreach my $job_file (@failed_jobs) { $job_file =~ /(\d+)\.job/; my $jobid = $1; - my ($status, $message) = $comm->send_command("SET_JOB_STATUS $jobid 'Failed'"); + my ($status, $message) = $comm->send_command("SET_JOB_STATUS", $jobid, "Failed"); open(FILE, $job_file) or die "Couldn't open job file $job_file: $!"; my $job_data = join("", ); close(FILE); @@ -214,7 +214,7 @@ sub scan_jobs_dir { foreach my $job_file (@done_jobs) { $job_file =~ /(\d+)\.job/; my $jobid = $1; - my ($status, $message) = $comm->send_command("SET_JOB_STATUS $jobid 'Done'"); + my ($status, $message) = $comm->send_command("SET_JOB_STATUS", $jobid, "Done"); } return @existing_jobs; -- cgit v1.2.3-65-gdbad From 13b6e17a910ca12433f77c7c4e62e3d1c1ef8a3e Mon Sep 17 00:00:00 2001 From: Andrew Gaffney Date: Sat, 12 Jan 2008 03:58:23 +0000 Subject: let's actually run the jobs we get...untested svn path=/branches/new-fu/; revision=350 --- client/scireclient.pl | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/client/scireclient.pl b/client/scireclient.pl index c054e17..a9cfe20 100755 --- a/client/scireclient.pl +++ b/client/scireclient.pl @@ -37,6 +37,24 @@ sub run_main { } } + my @new_jobs = glob("$conf{job_dir}/queue/*.job"); + for (@new_jobs) { + my $job = Scire::Job->new(); + $job->load_jobfile($_); + $job->set_stdout_file("$conf{job_dir}/queue/$job->{jobid}.out"); + $job->set_stderr_file("$conf{job_dir}/queue/$job->{jobid}.err"); + $job->set_script_file("$conf{job_dir}/queue/$job->{jobid}.script"); + my $exitcode = $job->run(); + if(!$exitcode) { + # Successful job completion + system("mv $conf{job_dir}/queue/$job->{jobid}.* $conf{job_dir}/done/"); + } else { + # Job failed + system("mv $conf{job_dir}/queue/$job->{jobid}.* $conf{job_dir}/failed/"); + } + } + + talk_to_server(); } sub talk_to_server { @@ -196,10 +214,14 @@ sub get_jobs { sub scan_jobs_dir { #Scan the dirs for job files. - my @existing_jobs = glob("$conf{job_dir}/queue/*"); - my @failed_jobs = glob("$conf{job_dir}/failed/*"); - my @done_jobs = glob("$conf{job_dir}/done/*"); + my @existing_jobs = glob("$conf{job_dir}/queue/*.job"); + my @failed_jobs = glob("$conf{job_dir}/failed/*.job"); + my @done_jobs = glob("$conf{job_dir}/done/*.job"); + # XXX: this function should just scan the various job dirs, create a Scire::Job object + # for each job found, and return a structure containing the info, so that another + # function can act on the completed jobs + #Report on those jobs needing reporting. foreach my $job_file (@failed_jobs) { $job_file =~ /(\d+)\.job/; @@ -215,6 +237,7 @@ sub scan_jobs_dir { $job_file =~ /(\d+)\.job/; my $jobid = $1; my ($status, $message) = $comm->send_command("SET_JOB_STATUS", $jobid, "Done"); + # XXX: Send job output } return @existing_jobs; -- cgit v1.2.3-65-gdbad From a519d213fa9af075d9d94dbec990ab85fc34d5cf Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 17 Feb 2008 19:40:38 +0000 Subject: removing some comments. small other touchups. svn path=/branches/new-fu/; revision=357 --- server/scireserver.pl | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/server/scireserver.pl b/server/scireserver.pl index fa5de61..1af39ee 100755 --- a/server/scireserver.pl +++ b/server/scireserver.pl @@ -55,7 +55,7 @@ my $dbh = DBI->connect($connect_string, $conf{db_user}, $conf{db_passwd}, { Rai while(<>) { my ($command, @args) = parse_command($_); # chomp( my $line = $_); -# debug("DEBUG: line is: $line"); + debug("DEBUG: line is: $_"); # SEE http://agaffney.org/mediawiki/index.php/SSH-based_protocol for documentation on the protocol. if($command eq "QUIT") { @@ -97,7 +97,7 @@ while(<>) { my @filenames = ("$conf{job_dir}/$client_id/result/$jobid.stdout", "$conf{job_dir}/$client_id/result/$jobid.stderr"); print "OK " . join(" ", @filenames) . "\n"; } elsif ($command eq "JOBFILE_SENT") { - my @filenames = @args; + my @jobfiles = @args; process_jobfile($_) foreach(@jobfiles); print "OK\n" } else { @@ -144,9 +144,6 @@ sub register_client { eval { run_query('LOCK TABLES `gacl_axo_seq` WRITE'); -# debug("Query is $query"); - #execute it -# $dbh->do($query); $query = 'SELECT id FROM `gacl_axo_seq`'; $sth = run_query($query); $id = $sth->fetchrow_hashref->{'id'}; @@ -154,8 +151,6 @@ sub register_client { $query = 'UPDATE `gacl_axo_seq` SET id=?'; run_query($query,$id); run_query('UNLOCK TABLES'); -# debug("Query is $query"); -# $dbh->do($query); }; ($@) and print "ERROR during fetching of id sequence: $DBI::errstr\n"; @@ -165,7 +160,6 @@ sub register_client { #NOTE: not sure if this query is still valid. may be using id instead of hostname for one of those two now. $query = 'INSERT INTO clients (clientid,digest,hostname,mac,ip,status) VALUES (?,?,?,?,?,?)'; - #execute with $id, client_cert.digest("sha1"),crypto.dump_certificate(crypto.FILETYPE_PEM,client_cert),$hostname,$mac,$ip,$status_id)) run_query($query,$id,$digest,$hostname,$mac,$ip,$status_id); }; ($@) and print "ERROR Could not insert client with $query: $DBI::errstr\n"; @@ -183,14 +177,11 @@ sub identify_client { my $query = 'SELECT client_status.statusname, clients.clientid FROM clients JOIN client_status on (clients.status = client_status.statusid) WHERE clients.digest=?'; my $sth = run_query($query,$digest); - #debug("Query is $query"); - #my $sth = $dbh->prepare($query); - #$sth->execute($digest); my $hashref = $sth->fetchrow_hashref(); debug(Dumper($hashref)); my $status_name = $hashref->{'statusname'}; $client_id = $hashref->{'clientid'}; - if (defined($client_id) and $client_id > 0) { #and ($status_name eq 'Active') { + if (defined($client_id) and ($client_id > 0) and ($status_name eq 'Active')) { $identified = 1; print "OK\n"; } else { @@ -216,9 +207,6 @@ EndOfQuery #FIXME ADD JOB DEPENDENCIES TO THIS QUERY. my $sth = run_query($query,$client_id); -# debug("Query is $query"); -# my $sth = $dbh->prepare($query); -# $sth->execute($client_id); my $jobs_ref = $sth->fetchall_arrayref(); # Don't ask me...ask the guys in #perl :P my @jobs = map { @$_ } @$jobs_ref; -- cgit v1.2.3-65-gdbad From 54769277dadda1dab75770bbafda97419f1baffe Mon Sep 17 00:00:00 2001 From: Preston Cody Date: Sun, 17 Feb 2008 19:41:55 +0000 Subject: trying to get this a bit more workable. svn path=/branches/new-fu/; revision=358 --- client/test.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/test.pl b/client/test.pl index c198964..e5fdcbb 100644 --- a/client/test.pl +++ b/client/test.pl @@ -9,11 +9,11 @@ use Scire::Job; use Scire::Communicator; my $job = Scire::Job->new(); -$job->load_jobfile("/tmp/scirejobs/job1.job"); +$job->load_jobfile("/tmp/scirejobs/queue/39.job"); #print $job->{script_data} . "\n"; -$job->set_stdout_file("/tmp/scire_stdout.txt"); -$job->set_stderr_file("/tmp/scire_stderr.txt"); -$job->set_script_file("/tmp/scirejobs/runjob.sh"); +$job->set_stdout_file("/tmp/scirejobs/result/39_stdout.txt"); +$job->set_stderr_file("/tmp/scirejobs/result/39_stderr.txt"); +$job->set_script_file("/tmp/scirejobs/run/runjob.sh"); my $exitcode = $job->run(); print "Job complete with exit code ${exitcode}\n"; -- cgit v1.2.3-65-gdbad