
#
# The comms-backup will probe your comms daemon every 30 minutes
# to check that it is still alive.  The frequency of these
# checks is specified in your crontab file.  This comms-backup
# uses the client script to connect to the port specified
# in the file "./proc/<hostname>-port" that was created by 
# your comms daemon.  If the comms daemon does not respond
# appropriatly, or does not respond at all, then it will be 
# restarted.
#

$HOME       = $ARGV[0];
$ID         = $ARGV[1];
$HOST       = $ARGV[2];
$NOTIFY     = $ARGV[3];
$SERVER     = $ARGV[4];
$CACHE      = $ARGV[5];
$ENV{CACHE} = $CACHE;
$| = 1;

if (!$HOME|!$ID|!$HOST|!$SERVER|!$CACHE) {
  print "Usage: comms-backup.pl <comms. home> <solver id> <host> <contact> <server>\n";
  exit(-1); 
}

#
# Port comms daemon should be on.
#

unless (chop($PORT = `cat $CACHE/proc/${ID}_${HOST}\@$SERVER.port`)) {
  &restart ("comms-backup.pl: Error reading $CACHE/proc/${ID}_${HOST}\@$SERVER.port");
} 

#
# Attempt "verify" command <numAttempt> times
#

$OK = 0;
$numAttempt = 6;
$expecting = "$ID\@$SERVER";
for ($i=0;$i<$numAttempt;$i++) {
  chop($result = `echo verify|$^X $HOME/bin/client.pl -silent -manual -server $HOST:$PORT -alarm 10`);
  if ($result ne $expecting) {
    print "comms-backup.pl: reply expected: \"$expecting\"\n";
    print "comms-backup.pl: reply received: \"$result\"\n";
    chop($date = `date`); 
    printf("comms-backup.pl: $date: attempt $i failed\n");
  } else {
    $OK = 1;
    last;
  }
} 

&restart("\"$ID\@$SERVER\" failed to respond") unless ($OK);
@date = localtime(time);
if($OK and ($date[3]==1 || $date[3]==16)) {
    if($date[2] < 1 && $date[1] < 30) {
	&restart("Bi-monthly restart of \"$ID\@$SERVER\"");
    }
}

#
# If "verify" succeeded, check that it returned the correct ID 
#

if ($result ne "$ID\@$SERVER") {
  &restart("comms-backup.pl: Expected \"$ID\@$SERVER\" got \"$result\"");
} 

#
# Subroutine for restarting the server
#

sub restart {

  chomp($date = `date`); 
  print "comms-backup.pl: $date: $_[0]\n";

  #
  # Kill old server
  #

  open(PID, "$CACHE/proc/${ID}_${HOST}\@$SERVER.pid") or die "Could not open $CACHE/proc/${ID}_${HOST}\@$SERVER.pid\n";
  chomp($PID = <PID>);
  chomp($CDfile = <PID>);
  close PID;

  #
  # Kill entire process group, should not affect forked
  # children handling requests because they will have 
  # changed their group pid
  #

  kill -9, $PID;
  kill 9, $PID;  # in case we're on a system where the setpgrp failed

  unlink "/tmp/comms-daemon-$CDfile" or print "Could not remove old comms daemon file from tmp directory.\n";

  #
  # Restart server 
  #

  if(open(COMMAND, "$CACHE/proc/${ID}_${HOST}\@$SERVER.command")) {
      chomp($COMMAND = <COMMAND>);
      close COMMAND;
  } else {
      &notify("Error reading \"$CACHE/proc/${ID}_${HOST}\@$SERVER.command.\"") if $NOTIFY;  
      print "comms-backup.pl: Error reading \"$CACHE/proc/${ID}_${HOST}\@$SERVER.command.\"\n\n";
      exit(-1); 
  }

#
# Get rid of symbols that could make the regular expression match fail
#

  $COMMAND =~ s/^\s+//;		
  $COMMAND =~ s/\s+$//;
  $COMMAND =~ s/\^/\\\^/g;
  $COMMAND =~ s/\$/\\\$/g;
  $COMMAND =~ s/\+/\\\+/g;
  $COMMAND =~ s/\(/\\\(/g;
  $COMMAND =~ s/\)/\\\)/g;
  $COMMAND =~ s/\[/\\\[/g;
  $COMMAND =~ s/\]/\\\]/g;

  foreach $entry (@processes) {
      if($entry =~ m#(\d+).*$COMMAND#) {
	 kill 9, $1;
     }
  }


  $COMMAND =~ s/-nocron//g;
  system("$COMMAND -nocron > $CACHE/logs/${ID}_${HOST}\@$SERVER 2>&1  &");
  exit(0);
}

#
# Subroutine for notifying someone of the restart via e-mail
#

sub notify {
  open(handle, "|mail $NOTIFY");
  if ($_[0] =~ /./) {
    print handle "***NEOS-COMM ERROR***:\n\n";
    print handle "YOU MUST MANUALLY RESTART THE FOLLOWING SERVER:\n\n";
    print handle "server: $SERVER\n";
    print handle "solver: $ID\n";
    print handle "host:   $HOST\n\n";
    print handle "Here is the cause of the error on $date:\n\n";
    print handle "$_[0]\n\n";
    print handle "For more details, see the logs:\n\n";
    print handle "$CACHE/logs/${ID}_${HOST}\@$SERVER.comms-backup\n\n";
    print handle "and\n\n";
    print handle "$CACHE/logs/${ID}_${HOST}\@$SERVER.\n\n";
  } else {
    print handle "**THIS IS ONLY A WARNING**:\n\n";
    print handle "The following server was restarted on $date:\n\n";
    print handle "server: $SERVER\n";
    print handle "solver: $ID\n";
    print handle "host:   $HOST\n\n";
    print handle "For more details, see the logs:\n\n";
    print handle "$CACHE/logs/${ID}_${HOST}\@$SERVER.comms-backup\n\n";
    print handle "and\n\n";
    print handle "$CACHE/logs/${ID}_${HOST}\@$SERVER.\n\n";
  }
  close (handle);
}

