#!/usr/bin/env perl
##########################################################################
#
# routine: podmov.pl
#
# purpose: Each whacked pod file, EDT set, or EPC file is moved to a pipeline
#          input directory for processing.  A rename is performed for pod
#          files to make sure that the pipeline doesn't see the new pod file
#          until it is ready for pick-up.  This is not needed for EDT files,
#          however an OSF must be created on the data reduction blackboard
#          for any EDT sets moved to the pipeline.  EPC files just need
#          a file move.
#
# input:   ENV.POD_RENAME_DIRECTORY - data reduction directory for PODs
#          ENV.EDT_RENAME_DIRECTORY - data reduction directory for EDT files
#          ENV.EPC_RENAME_DIRECTORY - data reduction directory for EPC files
#          ENV.DATA_REDUCTION_PATH - name of data reduction path
#          ENV.EDT_EXT_MASK - file extension mask to pick up EDT files
#          ENV.OSF_EDT_STATUS - value for EDT set OSF status columns
#
# modification history:
#
#   date    opr     who       reason
# -------- -----    --------- --------------------------------------
# 09/03/00 43165    MSwam     first version
# 06/20/01 44084    MSwam     verify file size after move 
# 08/20/01 43913_02 MSwam     replace start_status with edt_start
# 09/07/01 43913_04 Sherbert  change OSF_EDT_START to OSF_EDT_STATUS to 
#      ""    ""      ""       match resource files
# 10/08/01 44343    Sherbert  Shorten EDT trx line with a unix cmd
# 10/26/01   ""      ""       A little more clean up including deletion
#                             of a non-working block of Mike's
# 12/07/01 44598    MSwam     Set DATAID for EDT processing
# 05/07/02 45707    Sherbert  Take out linesizing; carefully split many long
#                             PrintMsg lines so that continued line begins at
#                             column 17. (Readability)
# 04/04/03 43915    WMiller   DADS 10.0 support
# 11/15/04 50663    MSwam     Add EPC file move, enable COS,WF3
# 06/30/10 44712    MSwam     Remove runsql include
# 
##########################################################################
#
# prepend the Perl include variable using all components of PATH
# (this allows us to reuse other OTFR Perl code)
#
$PATH  = $ENV{"PATH"};
@parts = split /:/, $PATH;
while (defined($val = pop(@parts))) {
   if (substr($val,0,1) eq "/") { unshift(@INC, $val)}
}

#
# include these modules
#
require 'printmsg.pl';       # prints a formatted status message
require 'resolve.pl';        # resolve stretches
require 'otfr_setup.pl';     # OTFR pipeline set-up for processing a request
require 'db_otfr_stats_mv.pl'; # update sci_proc_start_time in otfr_request_stats

OTFR_setup();

PrintMsg("I","podmov: Processing $p_dir");

# open the input file listing exposures and podfile names
$infile = $p_dir . ".podlist";
if (!open(PODLIST, "$infile")) {
   PrintMsg("E","podmov ERROR: failed to open infile $infile\n"); 
   exit $quit_this_dataset;
}

#
# obtain resource values from environment
$POD_RENAME_DIRECTORY = Resolve("POD_RENAME_DIRECTORY");
$EDT_RENAME_DIRECTORY = Resolve("EDT_RENAME_DIRECTORY");
$EPC_RENAME_DIRECTORY = Resolve("EPC_RENAME_DIRECTORY");
$DATA_REDUCTION_PATH  = $ENV{"DATA_REDUCTION_PATH"};
$EDT_EXT_MASK         = $ENV{"EDT_EXT_MASK"};
$OSF_EDT_STATUS       = $ENV{"OSF_EDT_STATUS"};
$MSG_REPORT_LEVEL     = $ENV{"MSG_REPORT_LEVEL"};    ## Creates uninitialized value when MSG_REPORT_LEVEL undefined
$try_again_later = 3;  # exit code when a rename collision occurs

##
## Add some trace lines to figure out where my OSF_EDT_STATUS value is going
##
if ($MSG_REPORT_LEVEL eq "MSG_ALL" ) {
    PrintMsg("D","podmov: Verifying MSG_REPORT_LEVEL    : $MSG_REPORT_LEVEL");
    PrintMsg("D","podmov: Verifying POD_RENAME_DIRECTORY: $POD_RENAME_DIRECTORY");
    PrintMsg("D","podmov: Verifying EDT_RENAME_DIRECTORY: $EDT_RENAME_DIRECTORY");
    PrintMsg("D","podmov: Verifying DATA_REDUCTION_PATH : $DATA_REDUCTION_PATH");
    PrintMsg("D","podmov: Verifying EDT_EXT_MASK        : $EDT_EXT_MASK");
    PrintMsg("D","podmov: Verifying OSF_EDT_STATUS      : $OSF_EDT_STATUS");
}

#
# Read each podlist entry and attempt to move files to the data reduction pipeline.
# Try a move for every PODLIST entry, even if one or more moves fail.
#
$failed_move = 0;
while (<PODLIST>) {
   #
   # split the read buffer ($_) into fields and lower case them
   ($exp, $pod, $indicator) = split(/ /,$_);  
   $exp = lc($exp);
   $pod = lc($pod);
   chomp($indicator);

   if ($indicator eq "EDT") {
       # move EDT set
       #
       # verify that destination directory and EDT files exist 
       if (! -d $EDT_RENAME_DIRECTORY) {
          PrintMsg("E","podmov ERROR: Unable to locate 
                EDT_RENAME_DIRECTORY $EDT_RENAME_DIRECTORY\n");
          exit $quit_this_dataset;
       }
       $edt_mask = $pod . $EDT_EXT_MASK;
       @edt = glob($edt_mask);
       if (! scalar(@edt)) {
          PrintMsg("I","podmov: No EDT files located using $edt_mask");
          #
          # lack of EDT files could indicate that the move already occurred
          # and this is a retry; process the next PODLIST record
          #
          next;
       }
       # files and directory found, 

       # move the files
       if (! transfer_fileset($edt_mask, $EDT_RENAME_DIRECTORY)) {
          # transfer failed, try next PODLIST entry
          $failed_move = 1;
          next;
       }
      
       # determine instrument by first character of ipppssoot
       if (substr($pod,0,1) eq "n") {
          $DATATYP = "nic";
       }
       elsif (substr($pod,0,1) eq "u") {
          $DATATYP = "wf2";
       }
       elsif (substr($pod,0,1) eq "o") {
          $DATATYP = "sti";
       }
       elsif (substr($pod,0,1) eq "j") {
          $DATATYP = "acs";
       }
       elsif (substr($pod,0,1) eq "l") {
          $DATATYP = "cos";
       }
       elsif (substr($pod,0,1) eq "i") {
          $DATATYP = "wf3";
       }
       else {
          PrintMsg("E","podmov ERROR: unknown instrument: $pod\n");
          exit $quit_this_dataset;
       }

       # create OSF in data reduction pipeline
       PrintMsg("I","podmov: osf_create -p $DATA_REDUCTION_PATH -f $pod -t $DATATYP -s $OSF_EDT_STATUS -n 000");
       `osf_create -p $DATA_REDUCTION_PATH -f $pod -t $DATATYP -s $OSF_EDT_STATUS -n 000`;
       if ($? != 0) {
           PrintMsg("E","podmov ERROR: Failed to create OSF 
                for $exp on $DATA_REDUCTION_PATH\n");
           exit $quit_this_dataset;
       }

       # process the next PODLIST record
       next;
   }

   # move a POD file
   #
   # form the output pod file name (e.g. lz_7eac_068_0000173142_n54h5dku.pod)
   # by concatenating the original pod filename and the exposure name
   #
   if ($indicator eq "POD") {
     $outpod = $pod . "_" . $exp . ".pod";
     PrintMsg("D","podmov: $outpod");

     # if POD file exists, try to move it
     if (-e $outpod) {
       #
       # move the new pod file to the pipeline input directory
       if (! transfer_new_pod($outpod, $POD_RENAME_DIRECTORY)) {
          # transfer failed, try next PODLIST entry
          $failed_move = 1;
          next;
       }
     } # end if exists outpod

     # process the next PODLIST record
     next;
   } # end if indicator is POD

   # move an EPC file
   #
   if ($indicator eq "EPC") {
       # verify that destination directory and EPC file exist 
       if (! -d $EPC_RENAME_DIRECTORY) {
          PrintMsg("E","podmov ERROR: Unable to locate 
                EPC_RENAME_DIRECTORY $EPC_RENAME_DIRECTORY\n");
          exit $quit_this_dataset;
       }
       $epc_mask = $pod . "*_epc.fits";
       @epc = glob($epc_mask);
       if (! scalar(@epc)) {
          PrintMsg("I","podmov: No EPC file located using $epc_mask");
          #
          # lack of EPC files could indicate that the move already occurred
          # and this is a retry; process the next PODLIST record
          #
          next;
       }
       # files and directory found, 

       # move the file
       if (! transfer_fileset($epc_mask, $EPC_RENAME_DIRECTORY)) {
          # transfer failed, try next PODLIST entry
          $failed_move = 1;
          next;
       }

       # process the next PODLIST record
       next;
   } # end if indicator is EPC

} # end while podlist
close(PODLIST);

#
# if anything failed to be moved, try again later (a collision probably occurred)
if ($failed_move) {
     PrintMsg("I","podmov: something failed to be moved; 
                a retry will occur later");
     exit $try_again_later;
}

#
# save time request was sent for science processing
use POSIX qw(strftime);
$start_time = strftime "%d-%b-%Y %T", localtime;

# update OTFR stats in database
$rootname = uc($p_dset);
DB_otfr_stats_mv($rootname,$p_requestid,$start_time);

# no failed moves, exit success
exit $all_is_well;

##########################################################################
#
# routine: transfer_new_pod
#
# purpose: Attempts to move a podfile to a processing directory.
#          The podfile is appended with a dangle (_hold) to avoid
#          immediate pick-up by any poller.  Once the pod file is
#          in place, the dangle is removed by a file rename.
#
# inputs:  outpod - name of podfile in current location
#          POD_RENAME_DIRECTORY - destination directory for podfile
#
# returns: 0 = file move failed
#          1 = file move succeeded
#
# modification history:
#
#   date    opr     who     reason
# -------- -----  --------  --------------------------------------
# 09/03/00 43165  MSwam     first version
# 
##########################################################################
sub transfer_new_pod
{
   my($outpod, $POD_RENAME_DIRECTORY) = @_;
   my($new_podspec, $pod_ready);   # LOCAL VARS
   
   # create the new pod name for the pipeline input directory 
   # using an altered name so it won't be picked up by a file poller until 
   # ready
   #
   # SPECIAL CASE: If the pod name begins with "W" it is an old-style
   #               pacor name, and should be renamed to DDF-style
   #
   if ($outpod =~ /^w.*/ ) {
       $new_podspec = $POD_RENAME_DIRECTORY . "/" . "lz_a2_" . $outpod . "_hold";
   }
   else {
       $new_podspec = $POD_RENAME_DIRECTORY . "/" . $outpod . "_hold";
   }

   # copy the new pod to the pipeline input directory
   #
   PrintMsg("D","transfer_new_pod: try copy $outpod $new_podspec");
   use File::Copy;
   if (!copy($outpod, $new_podspec)) {
      PrintMsg("E","podmov ERROR: copy from $outpod to $new_podspec failed; 
                (error=$!) .\n"
);
      return 0;
   }

   #
   # verify the sizes of the source and dest files match
   # (work-around for dropped blocks in acdsdops environment)
   #
   ($dontcare_dev,$dontcare_ino,$dontcare_mode,$dontcare_nlink,$dontcare_uid,
    $dontcare_gid,$dontcare_rdev,$src_size,$dontcare_atime,$dontcare_mtime,
    $dontcare_ctime,$dontcare_blksize,$dontcare_blocks) = stat($outpod);

   ($dontcare_dev,$dontcare_ino,$dontcare_mode,$dontcare_nlink,$dontcare_uid,
    $dontcare_gid,$dontcare_rdev,$dest_size,$dontcare_atime,$dontcare_mtime,
    $dontcare_ctime,$dontcare_blksize,$dontcare_blocks) = stat($new_podspec);

   if ($src_size != $dest_size) {
      PrintMsg("E","transfer_new_pod in podmov ERROR: File size verification failed: 
                $outpod:$src_size != $new_podspec:$dest_size \n");
      return 0;
   }

   #
   # rename pod file for pipeline pickup (remove the _hold suffix)
   $pod_ready = $new_podspec;
   $pod_ready =~ s/_hold//;

   PrintMsg("D","transfer_new_pod in podmov: make_pod_visible: 
                try mv $new_podspec $pod_ready");
   if (!rename($new_podspec, $pod_ready)) {
      PrintMsg("W","transfer_new_pod in podmov: pod rename failed");
      return 0;
   }
   #
   # check for original file existence, since the error status is 
   # apparently not set for a collision
   if (-e $new_podspec) {
      PrintMsg("W","transfer_new_pod in podmov: pod rename of $new_podspec failed, 
                try again later...");
      return 0;
   }

   PrintMsg("D","transfer_new_pod in podmov: pod $pod_ready awaiting pipeline pick-up");
   return 1;
  
}

##########################################################################
#
# routine: transfer_fileset
#
# purpose: Attempts to move a set of files to a directory.
#
# inputs:  fileset - filemask for a set of files
#          rename_directory - destination directory 
#
# returns: 0 = failed to perform file move
#          1 = file move succeeded
#
# modification history:
#
#   date    opr     who     reason
# -------- -----  --------  --------------------------------------
# 09/03/00 43165  MSwam     first version
# 08/21/03 48370  Sherbert  Add missing /bin/ qualifier to mv
# 
##########################################################################
sub transfer_fileset
{
   my($fileset, $rename_directory) = @_;
   
   # move the fileset to the new directory
   # mv is used to allow cross-device transfers, and the -i option
   # causes mv to fail if a file already exists with the desired name
   #
   PrintMsg("D","transfer_fileset in podmov: try mv $fileset $rename_directory");
   `/bin/mv -i $fileset $rename_directory`;
   if ($? != 0) {
      # move failed, try again later
      PrintMsg("W","transfer_fileset in podmov: fileset rename failed ($!), 
                try again later...");
      return 0;
   }

   # all moves successful
   PrintMsg("D","transfer_fileset in podmov: fileset $fileset awaiting pipeline pick-up");
   return 1;
}