#!/usr/bin/env perl
#----------------------------------------------------------------------------
#
# Name: exposure_times
#
# This script is designed to be run by any calibration script or a similar
# script that sets the working directory to the CAL directory. This  
# controlling script must also redirect the stdout messages to the trailer.  
# 
# This perl script verifies that the input environment variables are OK, then
# gets either the full filename of the raw science file from the command line,
# or the full filename of the FITS asn table that contains the exposure names.
# The rootname of the raw data or the association file must be the first nine
# characters of this filename.
# 
# If the pipeline path is not set for UPDATE_EXPOSURE_TIMES, then it 
# immediately returns a success status. If this is a test pipeline that 
# does not support database updates, then it exits with a warning.
#
# It verifies read-access to the raw science. It uses the listhead tool to read
# all the file headers. For every header record, it looks for EXPNAME, EXPSTART
# and EXPEND keywords and obtains the MJD value of the time keywords. For STIS
# headers, it also obtains the OBSMODE keyword.
#
# For exposures with multiple images that are not associated,
# then the start time is the EXPSTART of first image extension and the
# end time is the EXPEND of the last image extension. For STIS ACQ exposures
# for which the OBSMODE value is ACQ, the end time is padded by 30 seconds. For
# raw science associations, at each extension having a EXPEND keyword, it 
# parses the rootname from EXPNAME into programatic ids.
#
# For each exposure, it converts the MJD time values to standard OPUS date 
# format. It updates the qolink_sms start_time, end_time and time_type fields.
# After each successful updating of the database, with a path name not set to
# NONE, it looks for the FGS pipeline OSF for the jitter observation and 
# restarts the RCHECK process if necessary.
#
# If the input file is an association table, the table will be read by LISTASN
# and the member type is parsed to determine the entries that are exposures. 
# The full raw science exposure filename is created from the member name and
# that file is processed.
#
# Usage:  (both within script or interactively)
#    exposure_times.pl full_filename
#
#       example:
#
#         exposure_times.pl rootname_asn.fits   
#                        or
#         exposure_times.pl rootname_raw.fits   
#                  or, in the case of COS
#         exposure_times.pl ipppssoot_rawtag.fits OR 
#         exposure_times.pl ipppssoot_rawaccum.fits
#                     BUT NOT both
#
#    Important:
#    Call each STIS member individually (see bc_sti.csh for examples)
#
#    Note that the following environment variables (logicals) are 
#       always required: DSQUERY, and OPUS_DB. The pipeline ENV variable 
#       OK_TO_UPDATE_DATABASE is checked.
#
# History:
# Date     OPR      Who       Reason
# -------- -------- --------- ---------------------------------------------
# 10/03/01 44524    Baum      Initial version
# 04/26/02 45738    Baum      Use external subroutines.
# 05/17/02 45876    Baum      Use special STIS ACQ rule for start_time. 
#                             Make ProcessKeywords more efficient. Update
#                             description above.
# 10/25/02 46774    Baum      Fix bug in arguments to system call
# 10/08/03 49496    MSwam     Replace OTFR_WORLD with UPDATE_EXPOSURE_TIMES
# 11/13/03 49799    Baum      Use FGS_DATA_ID from ENV
# 05/08/06 55509    Sherbert  Remove inconsistencies with path defaults
# 03/06/08 44712    Sherbert  Move from ST_DBlib to DBI interface
# 07/10/08 60283    Sherbert  COS filenames not as simple as the rest
# 06/30/10 64432    MSwam     Use single quotes for SQLServer
#
#----------------------------------------------------------------------------

 # set up external routines
 unshift @INC,(split /:/, $ENV{PATH});
 require 'printmsg.pl';       # prints a formatted status message
 require 'do_dbi_pkg.pl';     # run query returning only record count
 require 'sogs_time_pkg.pl';  # manipulate time in SOGS format(yyy.ddd:hh:mm:ss) 

 #---------------------------------------------------------------------
 #specify exit status values
 #---------------------------------------------------------------------
 $PROCESS_FAILURE = 1; # exit status for CAL script
 $PROCESS_SUCCESS = 0; # exit status for CAL script

 #---------------------------------------------------------------------
 # exit with success status if UPDATE_EXPOSURE_TIMES is FALSE
 #---------------------------------------------------------------------
 $update_exposure_times = $ENV{"UPDATE_EXPOSURE_TIMES"};
 if ( ! defined ( $update_exposure_times ) ) {
    PrintMsg ( "E", "UPDATE_EXPOSURE_TIMES must be defined in environment." );
    exit ( $PROCESS_FAILURE );
 }
 PrintMsg ( "D", "UPDATE_EXPOSURE_TIMES is $update_exposure_times" . "." );
 if ( $update_exposure_times =~ /FALSE/ ) {
    PrintMsg ( "D", "Exit with success." );
    ## Comment out the following exit to test from command line
    ## IF you have UPDATE_EXPOSURE_TIMES set to FALSE
    exit ( $PROCESS_SUCCESS );
 }

 #---------------------------------------------------------------------
 #check for valid arguments
 #---------------------------------------------------------------------
 $num_arg = scalar @ARGV;
 if ($num_arg != 1) {
    PrintMsg ("E", "Bad usage: need one argument - full filename.");
    exit ( $PROCESS_FAILURE );
 }
 $fits_filespec = lc($ARGV[0]);

 #---------------------------------------------------------------------
 # check access to database
 #---------------------------------------------------------------------
 $OK_UPDATE = $ENV{"OK_TO_UPDATE_DATABASE"};
 if ( !defined($OK_UPDATE) ) { $OK_UPDATE = 'TRUE'; }
 if ( "T" ne uc ( substr ($OK_UPDATE,0,1) ) ) {
    PrintMsg ("W","OK_TO_UPDATE_DATABASE not TRUE.");
    PrintMsg ("W","This PATH must not be used in operations.");
    exit( $PROCESS_SUCCESS);
 }
 PrintMsg ("I", "--- start --- Exposure Times Updater -------");

 #---------------------------------------------------------------------
 # check pipeline variables to restart FGS pipeline
 #---------------------------------------------------------------------
 $fgs_path_name      = $ENV{"FGS_PATH_NAME"};
 if (defined($fgs_path_name)) {
    $fgs_data_id        = $ENV{"FGS_DATA_ID"};
    $fgs_rcheck_col     = $ENV{"FGS_RCHECK_COLUMN"};
    $fgs_hold_status    = $ENV{"FGS_HOLD_STATUS"};
    $fgs_restart_status = $ENV{"FGS_RESTART_STATUS"};

    if ((!defined($fgs_rcheck_col))   ||
        (!defined($fgs_data_id))      || 
        (!defined($fgs_hold_status))  || 
        (!defined($fgs_restart_status)))
    {
        PrintMsg("W","FGS_RCHECK_COLUMN, FGS_DATA_ID, FGS_HOLD_STATUS, " .
                     "and FGS_RESTART_STATUS not all defined.");
        PrintMsg("I","FGS Pipeline cannot be restarted."); 
        $restart = 0;
    } elsif ($fgs_path_name eq "NONE") {
        $restart = 0;  # this pipeline intentionally not accessing FGS
    } else {
        $restart = 1;
    }
 } else {
    $restart = 0;
 }
 if ( substr($fits_filespec, 10, 3) eq "asn" ) {
    UpdateAsnTimes( $fits_filespec );
 } else {
    UpdateExpTimes( $fits_filespec );
 }
 PrintMsg ("I","---  end  --- Exposure Times Updater -------");
 exit( $PROCESS_SUCCESS);
 #---------------------------------------------------------------------
 # end of main procedure -- subroutines follow
 #---------------------------------------------------------------------

 #---------------------------------------------------------------------
 #    subroutine to process association tables
 #---------------------------------------------------------------------
 sub UpdateAsnTimes {
    my ($asn_filespec) = @_;
    my @table;
    my $line;
    my $lines;
    my $num_exposures;
    my ($exposure,$present);
    my $exp_filespec;

    PrintMsg ("D", "started.", "UpdateAsnTimes" );
    #---------------------------------------------------------------------
    #verify input FITS file exists
    #---------------------------------------------------------------------
    if ( ! (-e $asn_filespec) ) {
        PrintMsg ("E", "Cannot find FITS file $asn_filespec");
        exit ( $PROCESS_FAILURE );
    }

    #---------------------------------------------------------------------
    # read table using listasn program
    #---------------------------------------------------------------------
    PrintMsg ("D", "Use listasn on $asn_filespec ", "UpdateAsnTimes" );
    @table = `listasn $asn_filespec -s`;
    if ($?) {
        PrintMsg ("E",
                  "Failure status returned from listasn for $asn_filespec", 
                  "UpdateAsnTimes" );
        exit ( $PROCESS_FAILURE );
    }
    $lines = scalar @table;
    PrintMsg ("D", "There are $lines lines in ASN table", "UpdateAsnTimes");
    if ($lines == 0) {
        PrintMsg ("E", "No output lines from asnlist tool." );
        exit ( $PROCESS_FAILURE );
    }

    #---------------------------------------------------------------------
    # check each line in the table for exposure records
    #---------------------------------------------------------------------
    $num_exposures=0;
    for ( @table ) {
        chomp;   # remove newline
        $line = $_;
        PrintMsg ("D", "ASN table line is $line", "UpdateAsnTimes");

        #-----------------------------------------------------------------------
        # extract the first nine characters of $line into $exposure and
        # after the occurence of EXP-(any non-white) and a space, extract either
        # the letter T or F into $present.
        #----------------------------------------------------------------------- 
        ( $exposure, $present ) = ( $line =~ /^(.........)\sEXP-\S+\s([TF])$/ );
        if ( defined ($present) ) {
            if ( $present eq "T" ) {
                $num_exposures++;
                $exposure = lc($exposure);
                ## COS breaks the pattern 
                ## and we want only one filename for $exposure
                $exp_filespec = findOneFile($exposure);
                UpdateExpTimes($exp_filespec);
            }
        }
    }
    if ( $num_exposures == 0 ) {
        PrintMsg ("E",
                "Did not find any exposures in the following listasn output" ); 
        for ( @table ) {
            $line = $_;
            print "$line\n";
        }
        PrintMsg ("E", "No qolink_sms times updated.");
        exit ( $PROCESS_FAILURE);
    }
 }

 #---------------------------------------------------------------------
 #   subroutine to process raw science file
 #---------------------------------------------------------------------
 sub UpdateExpTimes {
    my ($fits_filespec) = @_;
    my $rootname = substr ($fits_filespec, 0, 9);
    my $lines;
    my $line;
    my $db;
    my @header;
    my $associated;
    my $expname;
    my $start_mjd;
    my $end_mjd;

    #---------------------------------------------------------------------
    # get the external variables
    #---------------------------------------------------------------------
    my $DSQUERY =     $ENV{"DSQUERY"};
    my $OPUS_DB = lc ($ENV{"OPUS_DB"});

    #---------------------------------------------------------------------
    # begin processing
    #---------------------------------------------------------------------
    PrintMsg ("D", "started.", "UpdateExpTimes" );
    PrintMsg ("I","--- using: $fits_filespec");

    #---------------------------------------------------------------------
    #verify input FITS file exists, or quit with no error
    #---------------------------------------------------------------------
    PrintMsg ("D", "Verify input FITS file ($fits_filespec) exists.", 
                   "UpdateExpTimes" );
    if (!(-e $fits_filespec)) {
        PrintMsg ("W","Cannot find FITS file $fits_filespec");
        exit ( $PROCESS_SUCCESS );
    }

    #---------------------------------------------------------------------
    # open database for queries
    #---------------------------------------------------------------------
    $db = DoDBIopen ( $DSQUERY, $OPUS_DB, $PROCESS_FAILURE );

    #---------------------------------------------------------------------
    # read table using listhead program
    #---------------------------------------------------------------------
    PrintMsg ("D", "Use listhead on $fits_filespec ", "UpdateExpTimes" );
    @header = `listhead $fits_filespec -s`;
    if ($?) {
        PrintMsg ("E", 
                  "Failure status returned from listhead for $fits_filespec");
        exit ( $PROCESS_FAILURE );
    }
    $lines = scalar @header;
    PrintMsg ("D", "There are $lines lines in fits table", "UpdateExpTimes" );
    if ($lines == 0) {
        PrintMsg ("E","No output lines from listhead tool.");
        exit ( $PROCESS_FAILURE );
    }

    #---------------------------------------------------------------------
    # check each line in the table for  keywords
    #---------------------------------------------------------------------
    $associated = ('0' eq substr ($rootname,8,1) );
    $expname = $rootname;  # by default for non-associated datasets
    $is_stis = substr($rootname,0,1) eq "o";
    $is_acq = 0;
    PrintMsg ("D", 
              "associated: $associated, is_stis: $is_stis, is_acq: $is_acq", 
              "UpdateExpTimes" );
    
    for (@header) {
        $line = $_;
        $kw = substr ($line, 0, 9);
      
        if ( index("EXPNAME =EXPSTART=EXPEND  =OBSMODE =", $kw) >= 0 ) {  
            if ( 'EXPNAME =' eq $kw ) {
                $expname = substr($line,11,9); 
            } elsif ( 'EXPSTART=' eq $kw) {
                if ( ! defined($start_mjd) || $associated ) {
                    $start_mjd = substr($line,10,20);
                } 
            } elsif ( 'EXPEND  =' eq $kw ) {
                $end_mjd = substr($line,10,20); 
                if ( ! defined($start_mjd) ) {
                    PrintMsg ("E", "Missing EXPSTART keyword in header.");
                    exit( $PROCESS_FAILURE );
                }
                if ( $associated){
                    ProcessKeywords ($db, $expname, $start_mjd, $end_mjd, $is_acq);
                }
            } else {
                if ( $is_stis) {
                    PrintMsg ("D", "is STIS, check OBSMODE", "UpdateExpTimes" );
                    if ( 'OBSMODE =' eq $kw) {
                        $is_acq = ( substr($line,11,3) eq "ACQ" ) && 
                                  ( substr($line,14,1) ne "/" );
                    }
                    PrintMsg ("D", "is_acq: $is_acq", "UpdateExpTimes" );
                }  
            }
        }
    }
    if ( ! defined($end_mjd) ) {
        PrintMsg("E","Missing EXPEND keyword in header.");
        exit ( $PROCESS_FAILURE );
    }
    if  (! $associated ) {
        ProcessKeywords( $db, $expname, $start_mjd, $end_mjd, $is_acq );
    }

    #---------------------------------------------------------------------
    # end of all queries
    #---------------------------------------------------------------------
    DoDBIclose($db);
}

#---------------------------------------------------------------------
#   subroutine to update qolink_sms table and check FGS pipeline OSF
#---------------------------------------------------------------------
 sub ProcessKeywords 
 {
    my ($db,$rootname,$start_mjd,$end_mjd,$is_acq) = @_;

    PrintMsg ("D", "started.", "ProcessKeywords");
    #---------------------------------------------------------------------
    # If this is a STIS acq exposure, add 30 seconds to end time.
    #---------------------------------------------------------------------
    if ($is_acq) {
        $end_mjd = $end_mjd + (30.0/86400.0);
    }

    #---------------------------------------------------------------------
    # Convert MJD time to standard OPUS time format: yyyy.ddd:hh:mm:ss
    #---------------------------------------------------------------------
    $start_time = ConvertMJD ( $start_mjd ); 
    $end_time   = ConvertMJD ( $end_mjd ); 
    PrintMsg ("I", "For $rootname, start $start_time, end $end_time");
 
    #---------------------------------------------------------------------
    # Get programmatic ids for query
    #---------------------------------------------------------------------
    $program_id = uc ( substr($rootname,1,3) );
    $obset_id   = uc ( substr($rootname,4,2) );
    $ob_number  = uc ( substr($rootname,6,2) );

    #---------------------------------------------------------------------
    # update qolink_sms.start_time, end_time and time_type
    #---------------------------------------------------------------------
    $query = <<"EOQ";
UPDATE qolink_sms 
SET start_time='$start_time',end_time='$end_time',time_type='A'
WHERE program_id='$program_id' and
      obset_id='$obset_id' and
      ob_number='$ob_number'
EOQ

    $count = DoDBI( $db, $query );
    if ($count != 1) {
        PrintMsg ("E","Cannot update qolink_sms times for $rootname");
        exit( $PROCESS_FAILURE);
    }

    #---------------------------------------------------------------------
    # restart holding FGS OSF if in pipeline mode and FGS pipeline defined
    #---------------------------------------------------------------------
    if ($restart) {
        $fgs_root = substr ( $rootname, 0, 8 ) .'j';
        $osf_command = "osf_test -p $fgs_path_name -f $fgs_root " .
                       "-t $fgs_data_id -pr $fgs_rcheck_col";
        $osf_stat = `$osf_command`;
        if ($?) {
            PrintMsg ("I", 
                      "osf_test error for $fgs_root in path: $fgs_path_name");
        } else {
            $osf_stat= substr( $osf_stat, 0, 1 );
            if ( $fgs_hold_status eq $osf_stat ) {
                if ( $sys_stat = system("osf_update", 
                                        "-p", $fgs_path_name, 
                                        "-f", $fgs_root, 
                                        "-t", $fgs_data_id, 
                                        "-c", $fgs_rcheck_col, 
                                        "-s", $fgs_restart_status) ) {
                    $return_stat = $sys_stat / 256;
                    PrintMsg ("E", 
                    "Failed to update OSF for $fgs_root. Status $return_stat");
                    exit ( $PROCESS_FAILURE );
                } else {
                    my $msg = "Successful update of FGS OSF $fgs_root, " . 
                              "column $fgs_rcheck_col.";
                    PrintMsg ("I", $msg);
                }
            } elsif ( length ( $osf_stat ) == 1 ) {
                my $msg = "FGS OSF $fgs_root in path: $fgs_path_name, " .  
                          "$fgs_rcheck_col has status $osf_stat.";
                PrintMsg ("I", $msg);
            } else {
                my $msg = "FGS OSF $fgs_root not found in path: $fgs_path_name.";
                PrintMsg ("I", $msg);
            }
        }
    }
 }

#---------------------------------------------------------------------
#   subroutine to determine ONE filename per provided rootname
#   because COS can have multiple raw* types per rootname and does NOT
#   have simple rootname_raw.fits files
#---------------------------------------------------------------------
 sub findOneFile
 {
    my $inRootname = $_[0];

    PrintMsg ("D", "started.", "findOneFile");
    ## Judging from how it was done before, we are in the directory containing
    ## the rootnames, much as we are for calibration.  Therefore, no dirs need
    ## be worried about.  For calcos, I use unix-isms to find just one matching
    ## filename.  glob will allow me to do that in Perl.

    my $pattern = $inRootname . "_raw*.fits";
    my @rootMatchingFiles = glob($pattern);

    PrintMsg ("D", "Results of glob to find raw files: @rootMatchingFiles", 
              "findOneFile");

    return $rootMatchingFiles[0];
 }