#!/usr/bin/env perl
##########################################################################
#
# routine: reqcol.pl
#
# purpose: Checks the data reduction blackboard to see if all of
#          the needed exposures and product for an OTFR request have
#          completed calibration.  If so, they are copied to the OTFR
#          request-specific directory to prepare for their return
#          to DADS.
#
#          Also provides a collector timeout facility.  When the
#          collector is first triggered for a request, a file is placed
#          in the request-specific directory.  The age of this file
#          is checked at each collector wakeup, and if the needed
#          exposures and product never complete calibration by
#          the end of the timeout interval, collection is forced.
#          This picks up the files from the data reduction pipeline
#          in whatever state they happen to be.
#
# modification history:
#
#   date    opr     who        reason
# -------- -------- ---------- --------------------------------------
# 09/03/00 43165    MSwam      first version
# 02/14/01 43165_01 MSwam      adjust ASN ending character test   
# 04/11/01 43165_17 MSwam      differentiate timeout cases
# 11/26/01 44598_01 MSwam      parameterize calibration stage name
# 04/03/02 45084    MSwam      fix bug for handling singletons
# 04/04/03 43915    WMiller    DADS 10.0 support
# 08/21/03 48370    Sherbert   Add missing /bin/ qualifier to cp
# 06/30/10 44712    MSwam      Remove runsql include
# 
##########################################################################
#
# prepend the Perl include variable using all components of PATH
# (this allows us to reuse other OTFR Perl code)
#
$PATH  = $ENV{"PATH"};
@parts = split /:/, $PATH;
while (defined($val = pop(@parts))) {
   if (substr($val,0,1) eq "/") { unshift(@INC, $val)}
}

#
# include these modules
#
require 'printmsg.pl';       # prints a formatted status message
require 'resolve.pl';        # resolve stretches 
require 'otfr_setup.pl';     # OTFR pipeline set-up for processing a request
require 'db_otfr_stats_dc.pl';  # update sci_proc_stop_time in otfr_request_stats

OTFR_setup();

PrintMsg("I","Processing $p_dir");

# exit code for "timeout expired"
$flushit = 4;

# set-up 
#
$path = $ENV{"DATA_REDUCTION_PATH"};
$TIMEOUT_INTERVAL = $ENV{"TIMEOUT_INTERVAL"};
$CAL_STAGE_NAME = $ENV{"CAL_STAGE_NAME"};
$data_dir = Resolve("CAL_DIR");

# open the PODLIST file and read the shortened exposure names into a list
#
$infile = $p_dir . ".podlist";
if (!open(PODLIST, "$infile")) {
   PrintMsg("E","ERROR: failed to open infile $infile\n"); 
   exit $quit_this_dataset;
}
@exposures = ();
while (<PODLIST>) {
   #
   # split the read buffer ($_) into fields and lower case them
   ($exp, $dontcare_pod, $dontcare_indicator) = split(/ /,$_);  
   $exp = lc($exp);
   #
   # use only first 8 chars to avoid problems with last char
   $exp_short = substr($exp, 0, 8);
   push(@exposures, $exp_short);
}
close(PODLIST);
$ipppssoo = substr($p_dset, 0, 8);
$rootname = uc($p_dset);

# test for collector timeout file
#
$timeout_file = "$p_dir.timeout";
if (-e $timeout_file) {

   # get file modification time
   ($dontcare_dev,$dontcare_ino,$dontcare_mode,$dontcare_nlink,$dontcare_uid,$dontcare_gid,
    $dontcare_rdev,$dontcare_size,$dontcare_atime,$mtime,$dontcare_ctime,$dontcare_blksize,
    $dontcare_blocks) = stat($timeout_file);
            
   # convert to days and compare
   $age_secs = (time - $mtime);
   $age_mins = $age_secs / 60.0;
   $age_hours = $age_mins / 60.0;
   $age_days = $age_hours / 24.0;
   if ($age_days > $TIMEOUT_INTERVAL) {
       PrintMsg("W","TIMEOUT occurred; files will be flushed, as is.");
       #
       # try file copy from data reduction pipeline for each exposure
       foreach $exp (@exposures) {
           PrintMsg("I","copying files for $exp from $data_dir");

           # failures are possible if the data is stuck, so checking the status isn't needed
           `/bin/cp $data_dir/$exp* .`;
       }
       #
       # try file copy for the products (again, failure is possible if the data is stuck)
       PrintMsg("I","copying files for $ipppssoo from $data_dir");
       `/bin/cp $data_dir/$ipppssoo* .`;

       #
       # save time request was flushed
       use POSIX qw(strftime);
       $flush_time = strftime "%d-%b-%Y %T", localtime;

       # update OTFR stats in database
       DB_otfr_stats_dc($rootname,$p_requestid,$flush_time);

       # the request is now considered flushed
       #
       exit $flushit;
   }
}
else {
   # create an empty timeout file
   if (!open(TIMEOUT,">$timeout_file")) {
      PrintMsg("E","ERROR: Failed to create $timeout_file file\n");
      exit $quit_this_dataset;
   }
   close(TIMEOUT);
}

#
# define exit code for "results not ready yet"
$try_again_later = 3;

#
# search for completed calibration for each exposure
#
$missing = 0;
foreach $exp (@exposures) {
   #
   # get the calibration result for this exposure from the
   # data reduction BB
   @out = `osf_test -p $path -f '$exp?' -pr $CAL_STAGE_NAME`;
   $savestat = $?;
   if ($savestat != 0) {
      PrintMsg("E","ERROR: osf_test failed for $path, $exp; ecode= $savestat\n");
      exit $quit_this_dataset;
   }
   if (scalar(@out) == 0) {
      # no output means exposure is not ready
      $ca_value = " ";
   }
   else {
      # concatenate all OSF status values together
      # (q/s splits could have multiple OSFs in science pipeline)
      $ca_value = " ";
      for ($i=0; $i < scalar(@out); $i++) {
         $temp = $out[$i];
         # remove newline and extra space
         chomp($temp);
         chop($temp);
         $ca_value .= $temp;
      }
   }

   # match OSF status value to determine state of calibration
   # (expression is "any chars" + (c or n or f) + "any chars")
   if ($ca_value =~ /.*[cnf].*/ ) {
      PrintMsg("I","$exp: calibration COMPLETE, NOT PERFORMED or FLUSHED ($CAL_STAGE_NAME=$ca_value).");
   }
   else {
     # if any exposure is not ready, notify
     PrintMsg("I","$exp: calibration not ready; $CAL_STAGE_NAME =$ca_value");
     $missing = 1;
   }
}

# if any exposures are not yet ready, check again later
if ($missing) {
  exit $try_again_later;
}

# check for calibration completion of the exposure in the request name
# if it is an asn (otherwise it was already checked above)
#
# (NOTE: if the request is for a product (an ASN), the primary product
#        must be searched for in the data reduction pipeline, since
#        an OSF only exists for the primary product)
#
if ($p_dset =~ /.*[0-9a-i]$/) {          # product name (ends in 0-9 or a-i)
   $asn_id = $ipppssoo . '0';  # primary product ends in zero

   @out = `osf_test -p $path -f $asn_id -pr $CAL_STAGE_NAME`;
   $savestat = $?;
   if ($savestat != 0) {
      PrintMsg("E","ERROR: osf_test failed for $path, $asn_id; ecode= $savestat\n");
      exit $quit_this_dataset;
   }
   $testval = scalar(@out);
   if ($testval == 0) {
      # no output means exposure is not ready
      $ca_value = " ";
   }
   elsif ($testval > 1) {
      PrintMsg("E","Unexpectedly found $testval OSFs for $asn_id.");
      exit $quit_this_dataset;
   }
   else {
      $ca_value = $out[0];
      # remove newline and extra space
      chomp($ca_value);
      chop($ca_value);
   }

   if ($ca_value =~ /^[cnf]/ ) {
      PrintMsg("I","$asn_id: calibration COMPLETE, NOT PERFORMED, or FLUSHED ($CAL_STAGE_NAME=$ca_value).");
   }
   else {
      # if product is not ready, notify
      PrintMsg("I","$asn_id: ASN calibration not ready; $CAL_STAGE_NAME =$ca_value");
      exit $try_again_later;
   }
}

#
# copy the exposure files for this request from the data reduction
# pipeline directories to the local directory
#
foreach $exp (@exposures) {

   PrintMsg("I","copying files for $exp from $data_dir");

   # copy files from the pipeline directory
   `/bin/cp $data_dir/$exp* .`;
   if ($? != 0) {
      PrintMsg("E","ERROR: failed to copy $exp files from $data_dir\n");
      exit $quit_this_dataset;
   }
}

#
# copy product files from the pipeline directory
#
PrintMsg("I","copying files for $ipppssoo from $data_dir");
`/bin/cp $data_dir/$ipppssoo* .`;
if ($? != 0) {
   PrintMsg("E","ERROR: failed to copy $ipppssoo files from $data_dir\n");
   exit $quit_this_dataset;
}

#
# save time science processing was found to be complete
use POSIX qw(strftime);
$stop_time = strftime "%d-%b-%Y %T", localtime;

# update OTFR stats in database
DB_otfr_stats_dc($rootname,$p_requestid,$stop_time);

exit $all_is_well;