
#
# Communications daemon
#

use Socket;
use POSIX;

#
# change the process group for this process
# (also changes the group for all descendents).
# This way, the entire process group can be
# killed before restarting the comms daemon.
# Forked child processes that change their
# group process id to the pid of the child
# process should not be killed.
# This may not work on all systems.
# Therefore, we also kill the pid for insurance.
#

$result = setpgrp(0, $$);

#
# User Defaults
#

$CONTACT  = "";                 # e-mail address of contact person
$port     = 4000;               # Local port to accept connections on        
$SAVE     = 0;                  # 1 to save old jobs
$DEBUG    = 0;                  # 1 for debug mode 
$NOTIFY   = 0;                  # 1 to notify <contact> of jobs
$CRON     = 0;                  # 1 to enable cron backup
$progName = "comms-daemon.pl";  # progName
$MAXTIME  = 604000;             # kill job before communications expire 
$SIG{CHLD} = \&REAPER;          # avoid zombies on our administrator's machines
$fileLimit = 250000;            # max file size, generally in 512 byte blocks
$KILLABLE = 0;                  # solver job may be killed using ADMIN:KILLJOB

#
# Misc. 
#

sub _BSD{1};            # not sure why we need this...
$| = 1;                 # don't buffer stdout
$recvBufferSize = 8192; # for incoming jobs


#
# Parse command line arguments
#

foreach (@ARGV) {
  $ARGS .= "$_ ";
}
while ($_ = shift) {
  if (/usage/ or /help/) {
    &usage;
  } elsif (/^-cron$/) {
    $CRON = 1;
  } elsif (/^-nocron$/) {
    $CRON = 0;
  } elsif (/^-debug$/) {
    $DEBUG = 1;
  } elsif (/^-save$/) {
    $SAVE = 1;
  } elsif (/^-notify$/) {
    $NOTIFY = 1;
  } elsif (/^-type$/) {
    $TYPE = shift;
    $TYPE = "\U$TYPE";
    $TYPE =~ s/\s//g;
  } elsif (/^-server$/) {
    $SERVER = shift;
    unless ($SERVER =~ /(.+):(\d+)/) {
      print "\n$progName: Invalid server name: \"$SERVER\"\n"; 
      &usageUsage;
    } else {
      $server      = $1;
      $server_port = $2; 
    }
  } elsif (/^-solver$/) {
    $SOLVER = shift;
    $SOLVER = "\U$SOLVER";
    $SOLVER =~ s/\s//g;
  } elsif (/^-cache$/) {
    $CACHE = shift;
  } elsif (/^-port$/) {
    $port = shift;
  } elsif (/^-pass$/) {
    $PASSWORD = shift;
  } elsif (/^-contact$/) {
    $CONTACT = shift;
  } elsif (/^-driver$/) {
    $DRIVER = shift;
  } elsif (/^-alarm$/) {
    $MAXTIME = shift;
  } elsif (/^-flimit$/) {
    $fileLimit = shift;
  } elsif (/^-killable$/) {
    $KILLABLE = 1;
  } else {
    print "$progName: Unknown option \"$_\"\n";
    &usageUsage;
  }
}

#
# Data check
#

unless ($CACHE) {
  print "\n";
  print "$progName: You must specify the cache directory with -cache <dir>.\n";
  &usageUsage;
} 
unless ($TYPE) {
  print "\n";
  print "$progName: You must specify the solver type with -type <solver type>.\n";
  &usageUsage;
} 
unless ($SOLVER) {
  print "\n";
  print "$progName: You must specify the solver name with -solver <solver name>.\n";
  &usageUsage;
} 
unless ($PASSWORD) {
  print "\n";
  print "$progName: You must specify the password with -pass <password>.\n";
  &usageUsage;
} 
if (length($DRIVER)>0) {
  unless ($DRIVER =~ /^\//) {
    print "\n$progName: Please provide the absolute path of the driver.\n"; 
    &usageUsage;
  }
  unless ((-f "$DRIVER") && (-x "$DRIVER")) {
    print "\n";
    print "$progName: \"$DRIVER\" is not an executable file.\n";
    &usageUsage;
  } 
}
unless ($CONTACT) {
  print "\n";
  print "$progName: You must specify the contact person with -contact <e-mail address>.\n";
  &usageUsage;
} 
unless ($SERVER) {
  print "\n";
  print "$progName: You must specify the server to connect to with -server <host:port>.\n";
  &usageUsage;
} 


#
# Concatenate <solver type> and <solver name> to get ID
#

$ID = "$TYPE:$SOLVER";
$ID =~ s/\s//g;

#
# Make sure gzip and tar are in PATH (for restarts via cron which
# has a limited path)
#

chop($GZIP = `which gzip`);
$GZIP =~ s/\s//g;
unless (-x "$GZIP") {
  print "$progName: ERROR: \"gzip\" is not in your PATH.\n";
  exit(-1);
}
if ($GZIP =~ /\//) {
  $GZIP =~ /([^\/]+)$/;
  $GZIP =~ s/(.*)\/$1$/$1/;
  $gpath = $1;
}
chop($TAR  = `which tar`);
$TAR =~ s/\s//g;
unless (-x "$TAR") {
  print "$progName: ERROR: \"tar\" is not in your PATH.\n";
  exit(-1);
}
if ($TAR =~ /\//) {
  $TAR =~ /([^\/]+)$/;
  $TAR =~ s/(.*)\/$1$/$1/;
  $tpath = $1;
}

$CLIENT = "$^X $ENV{INSTALL_DIR}/bin/client.pl -server $server:$server_port";

if(open(PID, "$CACHE/proc/$ID_${LOCAL_HOST}\@$server:$server_port.pid")) {
  chomp($all_pid = <PID>);
  chomp($cdfile = <PID>);
  close PID;
  if(kill 0 => $all_pid) {
    print "Daemon running.\n";
    open(PORT, "$CACHE/proc/$ID_${LOCAL_HOST}\@$server:$server_port.port") or die "Could not open port file\n";
    chomp($port = <PORT>);
    close PORT;
    unless($ID =~ /ALL:ALL/) {  #Shouldn't be happening here
      &registerSolver();
    }
    exit 0;
  } else {
    unlink("/tmp/comms-daemon-$cdfile");
  }
}


#
# Create socket
#

$sockaddr = 'S n a4 x8';
$proto = getprotobyname('tcp')                             || die "getprotobyname: $!";
socket(Server, PF_INET, SOCK_STREAM, $proto)               || die "socket: $!";
setsockopt(Server, SOL_SOCKET, SO_REUSEADDR, pack("l", 1)) || die "setsockopt: $!";

#
# Bind to the first available port (starting at 4000) unless a
# specific port was specified on the command line. 
#

if ($PORT_SELECTED) {
  $paddr = pack($sockaddr, PF_INET, $port, INADDR_ANY);
  unless (bind(Server, $paddr)) {
    print "bind: port $port is not available.\n"; 
    print "$progName: Try selecting  another port with the -port<port> option.\n";
    exit(-1);
  }
} else {
  $MAX = 5000;
  for ($port = 4000; $port <=$MAX; $port++) {
    $paddr = pack($sockaddr, PF_INET, $port, INADDR_ANY);
    if (bind(Server, $paddr)) {
      last;
    } 
  }
  if ($port > $MAX) {
    print "bind: ports 4000 through $MAX are all busy.  Strange.\n"; 
    print "$progName: Try selecting  another port with the -port <port> option.\n";
    exit(-1); 
  }
}
listen(Server,SOMAXCONN) or die "listen: $!";

&registerSolver();

#
# Info for backup script
#

if(open(PORT, ">$CACHE/proc/${ID}_${LOCAL_HOST}\@$server:$server_port.port")) {
  print PORT "$port\n";
  close PORT;
} else {
  &error("Cannot create $port entry in $CACHE/proc/${ID}_${LOCAL_HOST}\@$server:$server_port.port");
}
if(open(PID, ">$CACHE/proc/${ID}_${LOCAL_HOST}\@$server:$server_port.pid")) {
  print PID "$$\n";
  if($0 =~ /comms-daemon-(\d+)/) {
    print PID "$1\n";
  } 
  close PID;
} else {
  &error("Cannot create $CACHE/proc/${ID}_${LOCAL_HOST}\@$server:$server_port.pid");
}
if(open(COMMAND, ">$CACHE/proc/${ID}_${LOCAL_HOST}\@$server:$server_port.command")) {
  print COMMAND "$ENV{INSTALL_DIR}/bin/comms-daemon $ARGS\n";
  close COMMAND;
} else {
  &error("Cannot create $CACHE/proc/${ID}_${LOCAL_HOST}\@$server:$server_port.command");
}

#
# Create a crontabfile for the crontab utility
#

if ($CRON) {
  srand(time|$$);
  $RANDOM = int(rand(30));
  @INDEX = ($RANDOM + 0, $RANDOM + 30); 

  #
  # First remove old crontab entry
  #

  if (!($LIST = `crontab -l 2>/dev/null`)) {
    &warning("\"crontab -l\" exited with an error.");
    $LIST = "";
  } else { 
    $LIST =~ s/###BEGIN NEOS CHECKER ENTRY \(token = $ID server = $SERVER\)###\n(.+)\n+###END NEOS CHECKER ENTRY \(token = $ID server = $SERVER\)###\n//i;

    #
    # Substitution intended to get rid of miles of "do not edit this file"s
    #
    $LIST =~ s/# DO NOT EDIT THIS FILE[^\n]*\n#[^\n]*\n#[^\n]*\n# DO NOT EDIT THIS FILE/# DO NOT EDIT THIS FILE/g;

  }

  #
  # Then add new one
  #

  $PERL = $^X;
  $CRONENTRY  = "$INDEX[0],$INDEX[1] * * * * $CACHE/bin/${ID}_${LOCAL_HOST}\@$server:$server_port.backup 2>/dev/null\n";
  $LIST .="###BEGIN NEOS CHECKER ENTRY (token = $ID server = $SERVER)###\n";
  $LIST .= $CRONENTRY;
  $LIST .="###END NEOS CHECKER ENTRY (token = $ID server = $SERVER)###\n";
  open(HANDLE, ">$CACHE/tmp/crontabfile-$LOCAL_HOST.tmp") or &warning ("Cannot create $CACHE/crontabfile.");
  print HANDLE $LIST;
  close(HANDLE);
  rename "$CACHE/tmp/crontabfile-$LOCAL_HOST.tmp", "$CACHE/tmp/crontabfile-$LOCAL_HOST";
  open(HANDLE, ">$CACHE/bin/${ID}_${LOCAL_HOST}\@$server:$server_port.backup");
  print HANDLE "#!/bin/sh\n";
  if ($ENV{HOST}) {
    print HANDLE "HOST=$ENV{HOST}; export HOST;\n";
  }
  if ($ENV{ARCH}) {
    print HANDLE "ARCH=$ENV{ARCH}; export ARCH;\n";
  }

  print HANDLE "PATH=$ENV{PATH}; export PATH;\n";
  print HANDLE "$PERL $CACHE/bin/comms-backup-1.0.pl $ENV{INSTALL_DIR} $ID $LOCAL_HOST $CONTACT $SERVER $CACHE >> $CACHE/logs/${ID}_${LOCAL_HOST}\@$server:$server_port.comms-backup 2>&1\n";
  close(HANDLE);
  chmod 0744, "$CACHE/bin/${ID}_${LOCAL_HOST}\@$server:$server_port.backup";

  #
  # Download crontab-backup.pl script
  #

  print "$progName: downloading comms-backup-1.0.pl\n";
  system("$CLIENT -request comms-backup-1.0.pl -file $CACHE/bin/comms-backup-1.0.pl") and ($waitedpid or &error("Could not run $CLIENT\n"));
  $waitedpid=0;

  #
  # Download disable-backup.pl script
  #

  print "$progName: downloading disable-backup-1.0.pl\n";
  system("$CLIENT -request disable-backup-1.0.pl -file $CACHE/bin/disable-backup-1.0.pl") and ($waitedpid or &error("Could not run $CLIENT\n"));
  $waitedpid=0;

  #
  # Activate crontab
  #

  system("crontab $CACHE/tmp/crontabfile-$LOCAL_HOST") and ($waitedpid or &error("Could not run crontab\n"));
  $waitedpid=0;
}

#
# Echo current settings 
#

if ($NOTIFY) {
  print "$progName: Notification of incoming jobs goes to $CONTACT.\n";
}
if ($DEBUG) {
  print "$progName: Debugging mode ON (STDERR will be displayed).\n";
}
if ($SAVE) {
  print "$progName: Old jobs will be saved in $CACHE/jobs.\n";
} 
if ($CRON) {
  print "$progName: New crontab activated.  Original saved in $CACHE/tmp.\n";
}

#
# Wait for connections from NEOS
#

sleep(1);
print "$progName: Daemon started. Waiting for connections.\n"; 
$waitedpid = 0;
for($con = 1;;$con++, $waitedpid=0) {

  #wait for connections or SIGCHLDs
  (($addr = accept(NS,Server)) || $waitedpid) or die $!;

  next if($waitedpid) and not $addr;

  $| = 1; select(NS); $| = 1; select(STDOUT);

  #
  # We do all of this before forking the child so
  # that the parent daemon can keep track of which
  # job number corresponds to which child pid.
  #

  ($af, $port, $inetaddr) = unpack($sockaddr, $addr);
  ($HOST) = gethostbyaddr($inetaddr, $AF_INET);
  @inetaddr = unpack('C4', $inetaddr);
  $IP = "";
  foreach $IP_PART (@inetaddr) {
    $IP .= "$IP_PART.";
  }
  chop($IP);
  $HOST = $IP;
  chop($ADDRESS = <NS>);
  $ADDRESS =~ s/(.*)\@.*/$1/; #Old clients give us the whole thing.
  chop($_ = <NS>); 
  $date = &date;


  if (($PID{$con} = fork()) == 0) {

    close Server;
    if(/^verify/) {
      print "V $date\n";
    } else {
      print "$ADDRESS\@$HOST:$date:$_\n";
    }

    #
    # Solver verification
    #

    if (/^verify/) {
      $data      = "$ID\@$server:$server_port\n";
      $length = length($data);
      print NS "$length\n";
      print NS $data;
      
    } elsif(/^check job (\d+)/) {
      
      #
      # check whether job's comms child is alive, then 
      # whether the solver driver is running
      #
      
      $jobNumber = $1;
      $childalive = 0;
      $driveralive = 0;
      print "Checking job $jobNumber\n";
      if(exists $JOB{$jobNumber}) {
        if(kill 0 => $JOB{$jobNumber}){
          $childalive = 1;
        }
      } else {
        if(open(README, "$CACHE/pids/$SERVER/$jobNumber.pid")) {
          chomp($readpid = <README>); 
          close README;
          if(kill 0 => $readpid){
            $childalive = 1;
          }
        }               
      }
      if(open(README, "$CACHE/pids/$jobNumber.dpid")) {
        chomp($driverpid = <README>);
        close README;
        if(kill 0 => $driverpid){
          $driveralive = 1;
        }              
      }
      if($driveralive) {
        $data = "Job $jobNumber alive\n";
        if(!$childalive) {
          if(chdir("$CACHE/jobs/$SERVER/$jobNumber")) {
            if(fork()==0) {
              &awaitDriver();
              exit 0;
            }
          } else {
            $data = "Job $jobNumber dead or finished\n";
            #
            # In this case, the driverpid we found may 
            # correspond to another process.
            #
          }
        }
      } else {
        $data = "Job $jobNumber dead or finished\n";
        if(chdir("$CACHE/jobs/$SERVER/$jobNumber")) {
          print "Attempting to return job $jobNumber results.\n";
          $data .= "Attempting to return results.\n";
          if(fork()==0) {
            &returnResults;
            exit 0;
          }
        } else {
          $data .= "Error: job directory no longer present.\n";
        }
      }
      
      $length = length($data);
      print NS "$length\n";
      print NS $data;

      #
      # Job to kill
      #
      
    } elsif (/^kill job (\d+)/) {

      $job2kill = $1;
      $data = "";

      if(!$KILLABLE) {
        $data .= "$progName: ADMIN:KILLJOB has been disabled on this solver.\n";
        print $data;
        $length = length($data);
        print NS "$length\n";
        print NS $data;
      } else {
        if(open(README, "$CACHE/pids/$SERVER/$job2kill.pid")){
          chomp($proc2kill = <README>);
          close README;
        }
        
        if(defined($proc2kill)) {
          $data = "$progName: Killing job $job2kill with pgid $proc2kill\n";
          print $data;
          $length = length($data);
          print NS "$length\n";
          print NS $data;     
          {
            local $SIG{"QUIT"} = 'IGNORE';
            kill -3, $proc2kill or print NS "$!: Couldn't kill child group\n";
          }
          unlink("$CACHE/pids/$job2kill.pid", "$CACHE/pids/$job2kill.dpid");
        } else {
          $data =   "$progName: No job $job2kill running (or job was started by previous solver daemon).\ncomms-daemon.pl: Job cannot be killed by Kill Job.\n";
          print $data;
          $length = length($data);
          print NS "$length\n";
          print NS $data;
        }
      }

      #
      # Job to execute
      #

    } elsif (/^begin job (\d+) (\d+)/) {
      print NS "-1\n" or &returnResults;   # the size of the data to be returned (unknown)
      $size       = $1;
      $jobNumber = $2;
      
      #
      #change the process group for this process
      #(also changes the group for all decendents)
      #

      setpgrp(0, $$);
      unless(-d "$CACHE/pids/$SERVER") {
        mkdir "$CACHE/pids/$SERVER", 0755 or &error("Could not make directory $CACHE/pids/$SERVER");
      }
      open(GPID, ">$CACHE/pids/$SERVER/$jobNumber.pid") or &warning("Could not write child pid to local file system.");
      print GPID $$;
      close GPID;
      
      #
      # Make working directory for job
      #        
      
      unless (-d "$CACHE/jobs/$SERVER") {
        mkdir "$CACHE/jobs/$SERVER", 0755 or &error("Could not make directory $CACHE/jobs/$SERVER");
      }
      mkdir "$CACHE/jobs/$SERVER/$jobNumber", 0755 or &error("Could not make directory $CACHE/jobs/$SERVER");
      chdir("$CACHE/jobs/$SERVER/$jobNumber");
      
      if ($NOTIFY) {
        print "$progName: notifying \"$CONTACT\"\n";
        open(MAILH, "|Mail -s '$ID' $CONTACT");
        print MAILH "$progName: executing $ID job#$jobNumber on $LOCAL_HOST\n"; 
        close(MAILH);
      }

      #
      # Read data.tar.gz from socket and write into dir.
      #
      
      print NS "$progName: downloading user data" or &returnResults;
      open(HANDLE, ">data.tar.gz");
      for ($i=0;($i+$recvBufferSize)<=$size;$i+=$recvBufferSize) {
        read(NS, $buffer, $recvBufferSize) or &error("Failed to read from socket.\n");
        print HANDLE $buffer;
        print NS "." or print "Socket closed at other end.\n";
      }
      if ($size%$recvBufferSize > 0) {
        read(NS, $buffer, $size%$recvBufferSize) or &error("Failed to read from socket.\n");
        print HANDLE $buffer;
      }
      close(HANDLE);
      
      #
      #   uncompress/untar data.tar.gz
      #
      
      print NS "\n$progName: uncompressing...\n" or print "Socket closed at other end.\n";
      system("gzip -d data.tar.gz") and ($waitedpid or &error("Could not run gzip -d on data.tar.gz\n"));
      
      print NS "$progName: untarring...\n" or print "Socket closed at other end.\n";
      system("tar -xf data.tar 2>&1") and ($waitedpid or &error("Could not run tar on data.tar\n"));
      
      #
      # Then invoke the software driver (i.e., solve script). 
      # All standard out from the solver goes back over
      # the socket connection.
      #
      
      print "$progName: starting \"$DRIVER\"\n";
      print NS "$progName: launching $ID driver...\n" or print "Socket closed at other end.\n"; 
      print NS "---------Begin Standard output/error--------\n" or print "Socket closed at other end.\n"; 
      unless (length($DRIVER)>0) {
        print "$progName: request for batch job $jobNumber, but no batch solver was specifed at startup\n";
      }
      if ($DEBUG) {
        $kidid = open (HANDLE, "ulimit -f $fileLimit; ulimit -c 0; $DRIVER $jobNumber 2>&1 |") or die "$progName: Cannot open software driver \"$DRIVER.\"\n";
      } else {
        $kidid = open (HANDLE, "ulimit -f $fileLimit; ulimit -c 0; $DRIVER $jobNumber 2>/dev/null|") or die "$progName: Cannot open software driver \"$DRIVER.\"\n";
      }
      if(open(PID, ">$CACHE/pids/$SERVER/$jobNumber.dpid")) 
      { 
        print PID $kidid;
        close PID;
      } else {
        &warning("Could not write driver pid to local file system.");
      }
      
# eval catches the die command, rather than letting it execute
# $@ is the value of returned by the eval
# $@ will eq "alarm clock restart" if the read timed out
# $@ eq null if the evaluation was parsed correctly
# $@ may still eq null even if there were errors from the read call
#
      $SNOOZED=0;
      $SNOOZETIME=14400;
      eval {
        local $SIG{"ALRM"} = \&snooze; 
        local $SIG{"QUIT"}  = sub {die "killed by admin";};
        if($MAXTIME > $SNOOZETIME) {
          alarm ($SNOOZETIME);
        } else {
          alarm ($MAXTIME);
        }
        $var = "";
        $readin = read(HANDLE, $var, 1);
        if(!(defined($readin))) {
          print NS "comms-daemon.pl: error with solver driver.\n";
          print "comms-daemon.pl: error with solver driver.\n";
          $readin = 0;
        }
        while ($readin > 0) {
          print NS "$var" or print "Socket closed at other end.\n";
          $readin = read(HANDLE, $var, 1);
          if(!(defined($readin))) {
            print NS "comms-daemon.pl: error with solver driver.\n";
            print "comms-daemon.pl: error with solver driver.\n";
            $readin = 0;
          }
        }       
        alarm (0);
      };
      if ($@ and $@ =~ /killed by admin/) { 
        local $SIG{HUP} = 'IGNORE';
        open(RESULT, ">>job.results") or print "$progName: Could not open results file.\n";
        print RESULT "\n\nJob killed by a NEOS Server admin via \"kill job\".\n";
        close RESULT;
        kill 'HUP' => 0 or print NS "$!: Couldn't kill child group\n" or print "Socket closed at other end.\n";
        sleep(1);        
      }
      elsif ($@ and $@ =~ /alarm clock/) { 
        eval {
          local $SIG{HUP} = 'IGNORE';
          local $SIG{INT} = 'IGNORE';
          local $SIG{QUIT} = 'IGNORE'; #So we cannot be kill-jobbed now
          print "\nJob $jobNumber timed out at solver max of $MAXTIME seconds.\n";
          kill 'INT' => 0 or (print NS "$!: Couldn't kill child group\n" and print "$!: Couldn't kill child group\n");
          sleep(100); #if it handles interrupts, give it time to clean up
          kill 'HUP' => 0 or (print NS "$!: Couldn't kill child group\n" and print "$!: Couldn't kill child group\n");
          kill 'KILL' => $kidid or (print "$!: OK: driver for job already dead or not accepting kill SIGKILL.\n" and print NS "$!: OK: driver for job already dead or not accepting kill SIGKILL.\n");
          print NS "Alarm sounded\n" or print "Socket closed at other end.\n";
          open(RESULT, ">>job.results") or print "Could not open results file.\n";
          print RESULT "\n\nJob timed out at solver max of $MAXTIME seconds.\n";
          close(RESULT);
        };
      }
      elsif($@) {
        open(RESULT, ">>job.results") or print "Could not open results file.\n";
        print RESULT "\n\nEval failure $@ in comms daemon.\n";
        print "\n\nEval failure $@ in comms daemon.\n";
        close RESULT;
      }
      close(HANDLE);
      print NS "---------End Standard output/error----------\n" or print "Socket closed at other end.\n"; 
      $SIG{HUP} = 'IGNORE';
      
      &returnResults;

      #
      # Ignore all other requests to this server
      #
      
    } else {
      $data    = "\"$_\" IGNORED\n";
      $length = length($data);
      print NS $data;
    }
    close(NS);
    exit(0);
  }

  if(/^begin job (\d+) (\d+)/) {
    $JOB{$2} = $PID{$con};
  }
  if(/^kill job (\d+)/)
  {
    if(exists $JOB{$2}) {
      delete $JOB{$2};
    }
  }
  
  close(NS);
}


#
# Usage subroutines
#

sub usageUsage {
  print "$progName: -usage for more information.\n\n";
  exit(-1);
}

sub usage{
  print "Usage: -type <solver type>\n";
  print "       -solver <solver name>\n"; 
  print "       -pass <password>\n"; 
  print "       -driver <filename>\n";
  print "       -contact <e-mail address>\n";
  print "       -cache <dir>\n";
  print "       -alarm <time limit> (seconds)\n";
  print "       -flimit <file size limit> (generally, 512 byte blocks)\n";
  print "      [-server <hostname:port>]\n";
  print "      [-port <local port>]\n";
  print "      [-notify]\n";
  print "      [-debug]\n";
  print "      [-save]\n";
  print "      [-cron]\n";
  print "      [-usage]\n";
  exit(0);
}


#
# Misc subroutines
#

sub error {
  if ($_[0] eq "ALRM") {
    print "$progName: **ERROR** Connection timed out. $_[0]\n";
  } else {
    print "$progName: **ERROR** $_[0]\n";
    
  }
  exit(-1);
} 

sub warning {
  print "$progName: WARNING: $_[0]\n";
} 

sub requires {
  print "$progName: $_[0] requires an argument.\n";
}

sub date {
  ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
  $year += 1900;
  $mon  += 1;
  $endDate = "$mon/$mday/$year";
  $endTime = "$hour:$min:$sec";
  return "$endDate $endTime";
}

sub REAPER {
  $waitedpid = wait;
  foreach $key (keys %PID) 
  {
    if ($waitedpid == $PID{$key}) 
    {
      delete $PID{$key};
    }
  }
  foreach $key (keys %JOB) {
    if ($waitedpid ==  $JOB{$key}) 
    {
      delete $JOB{$key};
    }
  }
  $SIG{CHLD} = \&REAPER;
  return 0;
}

sub returnResults {

  #
  # Include no printing to stdout, in case parent comms daemon 
  # has died (loses connection to file in some cases).
  #

  #
  # Finally return results in job.results to server
  #

  local $SIG{HUP} = 'IGNORE';
  $date = &date;      
  print NS "$progName: $date: returning job results\n";
  $waitServer=0;
  if (-s "job.results") {
    @statList = stat("job.results"); $size = $statList[7];
    $allvar = "";
    open (RHANDLE, "$CLIENT -alarm 60 -request  \"begin results $size $jobNumber\" < job.results|");
    while (read(RHANDLE, $var, 1) > 0) {
      $allvar = $allvar.$var;
    }
    close(RHANDLE) ;
    while($allvar =~ /ALARM/) {

# Give the Server a second chance to respond, after we sleep for a while

      $waitServer++;
      if($waitServer > 5) {
        last;
      }
      sleep(1200);
      $allvar = "";
      open (RHANDLE, "$CLIENT -alarm 60 -request  \"begin results $size $jobNumber\" < job.results|");
      while (read(RHANDLE, $var, 1) > 0) {
        $allvar = $allvar.$var;
      }
      close(RHANDLE);
    }
  } else {
    open(MAILH, "|Mail -s '$ID' $CONTACT");
    print MAILH "The following executable:\n\n\"$DRIVER\"\n\n";
    print MAILH "did not produce the file job.results for job #$jobNumber.\n"; 
      print MAILH "The job has been saved in the following directory:\n\n";
    print MAILH "$CACHE/jobs/$SERVER/$jobNumber\n";
    $SAVE = 1;
    close(MAILH);
  }
  
  #
  # Remove job directory if SAVE = 0 
  #
  
  chdir("$CACHE/jobs/$SERVER");
  if (!$SAVE) {
    print "$progName: Removing $CACHE/jobs/$SERVER/$jobNumber\n";
    @unlinks = ();
    if(defined($jobNumber) and ($jobNumber>=0)) { 
      @unlinks = <./$jobNumber/*>;
      foreach $unlink (@unlinks) {
        unlink $unlink;
      }
      rmdir "./$jobNumber";
    }
  } 

  # 
  # Remove pid files
  #
  unlink("$CACHE/pids/$SERVER/$jobNumber.pid", "$CACHE/pids/$SERVER/$jobNumber.dpid");
  exit 0;
    
}


sub registerSolver {

#
# Register selected local port with NEOS server
#

  print "$progName: Registering $ID on port $port.\n";
  open(HANDLE, "echo \"register $ID $PASSWORD $port\"|$CLIENT -alarm 60|");
  while (<HANDLE>) {
    if (/(.+): (.+) registered with port (\d+).$/) {
      $LOCAL_HOST = $1;
      # Get local host from hostname command instead of what
      # NEOS server gives us. 
      chop($LOCAL_HOST = `hostname`);
    } else { 
      print $_; 
    }
    $abort = 1 if (/ERROR/); 
  }
  close(HANDLE);
  if ($abort) {
    print "$progName: exited\n";
    exit(-1);
  }
}

sub awaitDriver {

  open(PID, ">$CACHE/pids/$SERVER/$jobNumber.pid") or die;
  print PID "$$";
  close PID;
  
  eval {
    local $SIG{"ALRM"} = sub {die "alarm clock"; };
    alarm(91600);
    while(kill 0 => $driverpid) {
      sleep 120;
    }
    alarm (0);
  };
  &returnResults;
}

sub snooze {
  local $extra;
  $SNOOZED++;
  if($SNOOZED*$SNOOZETIME >= $MAXTIME){
    die "alarm clock";
  }
  $extra = $MAXTIME - $SNOOZED*$SNOOZETIME;
  if($extra < $SNOOZETIME) {
    alarm($extra);
    print NS "Will check on remote comms script in another $extra seconds.\n";
  } else {
    alarm($SNOOZETIME);
    print NS "Will check on remote comms script in another $SNOOZETIME seconds.\n";
  }
}        
