#!/usr/bin/env perl 
#----------------------------------------------------------------------------
#
# Name: ingest_update_archive.pl
#
# This perl script is used for the UPDATE task of the Ingest pipeline. It is 
# designed to be run only as a command in an XPOLL process with no arguments.
# All parameters are passed as environmental variables. 
#
# Pipeline Usage:
#       command: ingest_update_archive.pl 
#     
#       In the pipeline, the resource file must set the following ENV 
#       variables: INGEST_LOG_DIR, PROBLEM_LOG_DIR,
#       UPDATE_STAGE, UPDATE_ALARM, UPDATE_BAD, UPDATE_READY,
#       OPUS_SERVER, SPSS_SERVER, ARCH_SERVER, 
#       OK_TO_UPDATE_DATABASE, and ARCH_DB.
#
#       This scripts uses the folowing ENV variables that are set by the 
#       XPOLL process: OSF_DATASET, OSF_DATA_ID, OSF_EVENT, and PATH_FILE.
#
#       There are optional pipeline specific ENV variables. The values of
#       INSTALL_ID.01, .02 , etc contained all the data_id vaues for OSFs that
#       are used for install requests. The values of the database field
#       ids_remote_pipeline must be provided as ENV variables that specify the
#       path file name for that pipeline. There is no required format for these
#       pipeline names but the resource file must specify all variables used for
#       the mission supported by the Ingest pipeline. For every mission
#       supported by the pipeline the resource file must have a parameter
#       with the name "<mission>_RELEASE_DELAY_MO". For example,
#       HST_RELEASE_DELAY_MO = 12 or FUSE_RELEASE_DELAY = 6.
#
# Implementation note; the PrintMsg subroutine prints to both the standard
# output and to a FILEHANDLE reference named $trl
#
# History:
# Date     OPR      Who         Reason
# -------- -------- ----------  ---------------------------------------------
# 01/08/04 49555    Baum        Initial code
# 03/02/04 50533    Baum        Add best version processing and allow
#                               group alarm processing even when there are
#                               no ingest_dat_set_info records.
# 04/13/04 50933    Baum        Add release_date processing and ads_data_source
#                               override parameters.
# 05/27/04 50933-01 Baum        Add PROPRIETARY keyword and set release date
#                               to getdate() for classes not in PROPRIETARY.
#                               Add MISSIONS keyword for pipelines supporting
#                               multiple missions.
# 03/18/05 53052    Baum        JBD project: remove NSA media fron queries
# 05/16/05 53531    Baum        Best version fix for HST CAL associations -add
#                               check_cal_dates and reset_cal_members - rename
#                               reset_cal_groups to reset_cal_dataset and
#                               make msg reporting conditional - make $ds_type_
#                               versions a global name and remove it from some
#                               argument interfaces.
# 06/03/05 53769    Baum        If PATH_NAME has no directory, accept the
#                               value as valid.
# 06/07/05 53785    Baum        Check ids_nsa_rsp_date while setting
#                               ids_install_flag to Y.  
# 11/02/09 61736    Sherbert    Q/S-split exception to precedence rules
# 11/20/09 61672    Sherbert    Proprietary Rights & get rid of globals, ugh.
# 03/24/10 64274    MSwam       Replace ST_DBlib with DBI
# 03/24/10 44712    MSwam       Replace ST_DBlib with DBI
# 04/26/10 64912    Sherbert    Q/S-split date inheritance 
# 09/17/10 65997    Sherbert    CSI/SSI should NOT send email
# 09/18/12 72255    Sherbert    get rid of DSQUERY
# 04/29/13 74104    MSwam       add insert_afi_nsa_info call for NewNSA
#----------------------------------------------------------------------------
 use strict 'vars';
 # set up external routines
 unshift @INC,(split /:/, $ENV{PATH});
 require 'printmsg.pl';              # prints a formatted status message
 require 'do_dbi_pkg.pl';
 require 'ingest_update_archive.pm'; # subroutines # use gave syntax error :(

 ## Testing
 #$ENV{"MSG_REPORT_LEVEL"} = "MSG_ALL" ;

 #prototype
 sub ErrorExit ($$) ;

 #constants
 my $true   = 1;
 my $false  = 0;

 # begin  Main
 #specify exit status values
 my $OSF_FAILURE =      7;   # exit status for XPOLL
 my $OSF_SUCCESS =      9;   # exit status for XPOLL
 my $OSF_DONE    =     11;   # exit status for XPOLL
 my $OSF_VAIT    =     13;   # exit status for XPOLL

 # Verify ENV variables set by XPOLL using pipeline
 my $osf_root    = $ENV{"OSF_DATASET"};
 my $osf_data_id = $ENV{"OSF_DATA_ID"};
 my $osf_event   = $ENV{"OSF_EVENT"};
 my $osf_path    = $ENV{"PATH_FILE"};
 my $sogs_disk   = $ENV{"SOGS_DISK"};
 # Verify ENV variables set by XPOLL using resource file
 my $ingest_log_dir   = $ENV{"INGEST_LOG_DIR"};
 my $problem_log_dir  = $ENV{"PROBLEM_LOG_DIR"};
 my $ok_to_update_db  = $ENV{"OK_TO_UPDATE_DATABASE"};
 my $update_stage     = $ENV{"UPDATE_STAGE"};
 my $update_alarm     = $ENV{"UPDATE_ALARM"};
 my $update_ready     = $ENV{"UPDATE_READY"};
 my $override_dir     = $ENV{"INGEST_SOURCE_DIR"};
 my $override_mask    = $ENV{"OVERRIDE_FILE_MASK"};
 my $override_req     = $ENV{"OVERRIDE_REQUIRED"};
 my $proprietary      = $ENV{"PROPRIETARY"};
 my $missions         = $ENV{"MISSIONS"};
 my $ds_type_versions = $ENV{"TYPE_VERSIONS"};
 my $to_address       = $ENV{"OPERATOR_MAIL_LIST"} ;

 my $OPUS_SERVER   = $ENV{"OPUS_SERVER"};       ## serves OPUS and ASSIST
 my $SPSS_SERVER   = $ENV{"SPSS_SERVER"};
 my $SPSS_DB       = $ENV{"SPSS_DB"};
 my $OPUS_DB       = $ENV{"OPUS_DB"};
 my $ASSIST_DB     = $ENV{"ASSIST_DB"};
 my $ARCH_DB       = $ENV{"ARCH_DB"};
 my $ARCH_SERVER   = $ENV{"ARCH_SERVER"};

 ## Satisfy strict
 my $release_delay_months ;
 my $prop_start_tm ;
 my $mission = "unknown" ;         # initialize just in case
 my $group_data_id = "unknown" ;   # initialize just in case
 my $group_name = "unknown" ;      # initialize just in case
 my $gen_date = "unknown" ;        # initialize for alarm mode
 my $pep_id = 0;                   # initialize 
 my $archive_class = 'N/A' ;       # initialize  in case group_osf=0
 my $remote_param_ref ;
 my %remote_params ;
 my $data_source ;
 my $var_name ;
 my $bad_data        = $false ;    # initialize (same truth as undef)
 my $completed_group = $false ;    # initialize (same truth as undef) 
 my $needs_insertion = $false ;    # initialize (same truth as undef)
 my $version ;

 ## Combine two different initial error messages into one
 if (!defined($osf_root)        || !defined($osf_data_id)      || 
     !defined($osf_event)       || !defined($osf_path)         || 
     !defined($sogs_disk)       || !defined($missions)         || 
     !defined($ingest_log_dir)  || !defined($problem_log_dir)  || 
     !defined($ok_to_update_db) || !defined($update_stage)     ||
     !defined($update_alarm)    || !defined($update_ready)     ||
     !defined($override_dir)    || !defined($override_mask)    ||
     !defined($override_req)    || !defined($proprietary)      ||
     !defined($OPUS_SERVER)     || !defined($ARCH_SERVER)      || 
     !defined($SPSS_DB)         || !defined($ARCH_DB)          || 
     !defined($ASSIST_DB)       || !defined($SPSS_SERVER)      || 
     !defined($to_address)      || !defined($ds_type_versions) 
    )  
 {
     PrintMsg("E", "Missing at least one ENV variable in resource file.");
     PrintMsg("E", "This script must be run as a pipeline command where ");
     PrintMsg("E", "OSF_DATASET, OSF_DATA_ID, PATH_FILE, SOGS_DISK and " );
     PrintMsg("E", "OSF_EVENT are defined.  Also need: ");
     PrintMsg("E", "INGEST_LOG_DIR, PROBLEM_LOG_DIR, OK_TO_UPDATE_DATABASE, ");
     PrintMsg("E", "UPDATE_STAGE, UPDATE_ALARM, UPDATE_READY, ");
     PrintMsg("E", "INGEST_SOURCE_DIR, OVERRIDE_FILE_MASK, MISSIONS, " ) ;
     PrintMsg("E", "PROPRIETARY, TYPE_VERSIONS, OPERATOR_MAIL_LIST, ") ;
     PrintMsg("E", "OVERRIDE_REQUIRED.  ...and need database locators: ");
     my $errMsg = "SPSS_SERVER, OPUS_SERVER, ARCH_SERVER, ";
     $errMsg = $errMsg . "SPSS_DB, ASSIST_DB, and ARCH_DB." ;
     ErrorExit( $errMsg, $OSF_FAILURE );
 }
 # must have valid directory or override file search failures will be
 #    misinterpreted as no files present
 if ( !-d $override_dir) {
     ErrorExit( "ENV variable INGEST_SOURCE_DIR is not a directory.", $OSF_FAILURE );
 }
 #verify that this pipeline allows database updates
 if ($ok_to_update_db ne "TRUE") {
     ErrorExit( "The value of OK_TO_UPDATE_DATABASE must be TRUE for this task", $OSF_FAILURE );
 }

 # strip out the directory spec from the $osf_path
 my $last_slash = rindex( $osf_path, "/");

 if ($last_slash >= 0) {
     $osf_path = substr( $osf_path, ($last_slash + 1));
 }

#put_install_id_list() ;    ## !! ONLY FOR COMMAND-LINE TESTING !!
 # Get list of INSTALL data_id values in ENV: INSTALL_ID.nn = <data_id>
 my @install_id ;
 eval { @install_id = get_install_id_list() } ;
 ErrorExit( $@, $OSF_FAILURE ) if $@ ;

 # open dataset log file for appending
 my $ds_log_spec = $ingest_log_dir."/".$osf_root."_".$osf_data_id.".log";
 open (TRL, ">>".$ds_log_spec);

 if (!(-w TRL)) {   # is it writeable
     ErrorExit( "Cannot open trailer $ds_log_spec", $OSF_FAILURE );
 }
 # create reference to the TRL filehandle for use by PrintMsg
 my $trl = \*TRL;
  
 # write signon message to process and trailer logs   
 PrintMsg ("I",
      "--- start --- Ingest Update Archive for $osf_root $osf_data_id ---");
 PrintMsg( 'I', "Recording updates to $ds_log_spec" );

 # determine if this pipelines has a data source override
 my @override_file_list = glob $override_dir.$override_mask;
 my $override_data_source = "unknown";
 my $number_of_files = scalar @override_file_list;
 if ($number_of_files == 0) {
    $override_data_source = $false;
    if ($override_req eq "Y") {
        ErrorExit( "Must provide source override file for this pipeline.", $OSF_FAILURE );
    }
 } elsif ($number_of_files == 1) {
    $override_data_source = $true;
    # get override data source name
    my $last_dot = rindex( $override_file_list[0], ".");
    if ($last_dot < 0) {
       ErrorExit("Invalid OVERRIDE_FILE_MASK is missing dot.", $OSF_FAILURE );
    }
    $data_source = uc(substr( $override_file_list[0], ($last_dot + 1)));
    my $data_source_len = length( $data_source);
    if ($data_source_len < 4 || $data_source_len > 6) {
       ErrorExit("Invalid length for data source override: $data_source", $OSF_FAILURE );
    }
    PrintMsg ("I", "NOTE: ads_data_source override detected: override ".
        "value is $data_source");
 } else {
    ErrorExit( "No more than one file may match $override_dir.$override_mask", $OSF_FAILURE );
 }
 # open database for queries
 my $db_arch = DoDBIopen ($ARCH_SERVER, $ARCH_DB, $OSF_FAILURE);
 my $db_spss = DoDBIopen ($OPUS_SERVER, $SPSS_DB, $OSF_FAILURE);
 my $db_assi = DoDBIopen ($OPUS_SERVER, $ASSIST_DB, $OSF_FAILURE);
 my $db_opus = DoDBIopen ($OPUS_SERVER, $OPUS_DB, $OSF_FAILURE);

 # Check if this OSF is used for an install request or if it is a group OSF
 PrintMsg( 'D', 'Check if this OSF is used for an install request or if it is a group OSF' );
 
 my $group_osf = $true;
 foreach my $data_id (@install_id) {
     if ($data_id eq $osf_data_id) {
        $group_osf = $false;
        last;
     }
 }
 PrintMsg( 'D', "group_osf is $group_osf. " );
 PrintMsg( 'D', "osf_event is $osf_event, used to determine process_mode " );
 my $process_mode = "UNDEFINED";
 # get processing mode by using EVENT_NAME
 if ($osf_event eq "OSF_TRIGGER1") {
    $process_mode = "UPDATE";
 } elsif ($osf_event eq "OSF_TRIGGER2") {
    $process_mode = "ALARM";
 } elsif ($osf_event eq "OSF_TRIGGER3") {
    $process_mode = "BAD_DATASET";
 } else {
    ErrorExit("Unexpected EVENT_NAME: $osf_event.", $OSF_FAILURE );
 }
 PrintMsg ("I", "This process has been triggered to use the $process_mode mode.");

 # get more parameters
 if ($group_osf) {
    PrintMsg( 'D', 'group_osf=true, Get group_osf parameters' );
    # first, set parameters to be used for all types of OSF
    PrintMsg( 'D', 'first, set parameters to be used for all types of OSF' );
    $group_name = $osf_root;
    $group_data_id = $osf_data_id;
    
    # second, get mission, archive_class and generation date of request
    PrintMsg( 'D', 'second, get mission, archive_class and generation date of request' );
    if (index($missions,",") < 0) {
       # there is only one mission, use the value in $missions
       $mission = $missions;
       PrintMsg( 'D', "there is only one mission, use the value $mission from missions $missions. " );
    } else {
       # use the dataid to query the ingest_up_control table
       PrintMsg( 'D', 'use the dataid to query the ingest_up_control table' );
       eval { ($mission) = get_mission_by_data_id( $db_arch, 
                                            $missions, $group_data_id ) } ;
       ErrorExit($@, $OSF_FAILURE ) if ($@) ;
       PrintMsg( 'D', "mission is $mission" );
    }
    if ($process_mode eq "UPDATE") {
       eval { ($gen_date,$archive_class) = get_gen_date_class( $db_arch, 
                                            $group_name, $group_data_id ) } ;
       ErrorExit( $@, $OSF_FAILURE ) if $@ ;
       my $msg = "gen_date: $gen_date; archive_class $archive_class." ;
       PrintMsg( 'I', $msg ) ;
    }
    # third, get remote pipeline parameters
    PrintMsg( 'D', 'third, get remote pipeline parameters ' );
   #eval { ($remote_path_file, $remote_stage, $remote_success, $remote_failure,
   #        $remote_alarm) = get_remote_parameters( $db_arch,
   #                                                $mission, $group_data_id ) } ;
    eval { ( $remote_param_ref ) = get_remote_parameters( $db_arch,
                                                    $mission, $group_data_id ) } ;
    ErrorExit( $@, $OSF_FAILURE ) if $@ ;
    %remote_params = %{$remote_param_ref} ;
    PrintMsg( 'D', "get_remote_parameters(...) gives us 
              \t\t remote_path_file $remote_params{remote_path_file}, 
              \t\t remote_stage     $remote_params{remote_stage},     
              \t\t remote_success   $remote_params{remote_success},   
              \t\t remote_failure   $remote_params{remote_failure},   
              \t\t remote_alarm     $remote_params{remote_alarm}. "
            );

    # fourth, get release delay months for this mission
    PrintMsg( 'D', 'fourth, get release delay months for this mission ' );
    $var_name = $mission."_RELEASE_DELAY_MO";
    $release_delay_months = $ENV{$var_name};
    if (!defined( $release_delay_months)) {
        ErrorExit( "Missing ENV for $var_name.", $OSF_FAILURE );
    }
    PrintMsg( 'D', "release_delay_months is $release_delay_months. " );
    ## PR 61627 ... want to create a hash with relevant information
    ## so move this determination here.  Do not fail if there is no pep_id
    ## Only need to look if this is a group_osf (an observation for ADS)
    ## otherwise can keep default which is set to 0
    eval { $pep_id = get_pep_id( $db_spss, $group_name, $ds_type_versions ) } ;
    ErrorExit( $@, $OSF_FAILURE ) if $@ ;
    PrintMsg( 'D', "pep_id is $pep_id" );
    ## sometimes the concept of proposal_id makes no sense, so nevermind:
#   if ( $pep_id == 0 ) { PrintMsg( 'W', "Invalid proposal_id: $pep_id" ) ; }
 } else {  # this is not a group OSF
    PrintMsg( 'D', 'group_osf=false (or !=true): this is not a group OSF ' );
    # first, get group identifiers
    PrintMsg( 'D', 'first, get group identifiers ' );
    eval { ($group_name, $group_data_id, $gen_date, $mission) = 
           get_group_identifiers( $db_arch, $osf_root ) } ;
    ErrorExit( $@, $OSF_FAILURE ) if $@ ;
    my $msg = "group_name is $group_name; group_data_id is $group_data_id; ";
    $msg = $msg . "gen_date is $gen_date; mission is $mission" ;
    PrintMsg( 'D', $msg ) ;
    
    # second, update status in database
    PrintMsg( 'D', 'second, update status in database' );
    eval { update_install_flag( $db_arch, $process_mode, $osf_root ) } ;
    ErrorExit( $@, $OSF_FAILURE ) if $@ ;
 }  ## end if group_osf
 # at this point $mission, $gen_date, $group_name and $group_data_id 
 PrintMsg( 'D', 'at this point $mission, $gen_date, $group_name, and $group_data_id ' );
 PrintMsg( 'D', "($mission, $gen_date, $group_name, and $group_data_id) " );
 # are defined for later use in any processing mode
 PrintMsg( 'D', 'are defined for later use in any processing mode ' );
 ## Create a hash to pass around, thus making it less likely necessary info 
 PrintMsg( 'D', 'Create a hash containing them and pep_id, trl, archive_class, misson:' );
 PrintMsg( 'D', "($pep_id, $trl, $archive_class, $mission)" );
 ## is missing when needed in subroutines.  2010/01/07
 my %group_info = (                   ## begin group_info def
      group_name    => $group_name    ## group_info
    , group_data_id => $group_data_id ## group_info
    , gen_date      => $gen_date      ## group_info
    , proposal_id   => $pep_id        ## group_info
    , trl           => $trl           ## group_info
    , archive_class => $archive_class ## group_info
    , mission       => $mission       ## group_info
 ) ;                                  ## end group_info def
 # To reference a hash, pass \%group_info
 # To deref the hash, do: my $hash_ref = @_; my %group_info = %{$hash_ref}
 # To access the parts, after deref: $group_info{group_name} (no quotes)
#require 'print_utils.pl' ; ## Del me
#printHash( %group_info );  ## Del me

 ## I think I can move this here, delete both places below.  Do so after test.
 # copy bad dataset log, if nec
#PrintMsg( 'D', 'Does a bad dataset log need to be copied?' );
#if ($process_mode eq "BAD_DATASET") {
#    PrintMsg( 'D', 'process_mode is BAD_DATASET ' );
#    copy_log_file( \%group_info, $problem_log_dir, $ds_log_spec );
#}
 
 # signal immediately if group_osf is in alarm mode
 PrintMsg( 'D', 'Is group_osf in alarm mode? ' );
 if ($group_osf && $process_mode ne "UPDATE") {
    my $msg = 'group_osf=true AND ' ;
    $msg = $msg . 'process_mode is either (ALARM or BAD_DATASET) (i.e. NOT UPDATE)' ;
    PrintMsg( 'D', $msg ) ;
    PrintMsg( 'I', 'group_osf IS in an alarm mode ' );
    # copy bad dataset log
    if ($process_mode eq "BAD_DATASET") {
       PrintMsg( 'D', 'process_mode is BAD_DATASET ' );
       PrintMsg( 'D', 'copy bad dataset log ' );
       eval { copy_log_file( \%group_info, $problem_log_dir, $ds_log_spec ) } ;
       ErrorExit( $@, $OSF_FAILURE ) if $@ ;
    }
    if ($remote_params{remote_path_file} ne "NONE") {
       eval { send_remote_status( \%group_info, $completed_group, 
                                  $process_mode, $needs_insertion, 
                                  $bad_data, \%remote_params ) } ;
       ErrorExit( $@, $OSF_FAILURE ) if $@ ;
       my $msg = "...called send_remote_status ( \%group_info, " ;
       $msg = $msg . "$completed_group, $process_mode, $needs_insertion, " ;
       $msg = $msg . "$bad_data, \%remote_params ) " ;
       PrintMsg( 'D', $msg );
    } else {
       PrintMsg( 'D', 'Cannot signal: ' );
       PrintMsg ("I", "No remote pipeline for this data_id ($group_data_id).");
    }
    PrintMsg ("I",
      "---  end  --- Ingest Update Archive for $osf_root $osf_data_id ---");
#   print "## TRACE exit OSF_DONE ( $OSF_DONE ) \n" ;
    exit( $OSF_DONE ) ;
 }
 # copy bad dataset log, if nec
 PrintMsg( 'D', 'Does a bad dataset log need to be copied?' );
 if ($process_mode eq "BAD_DATASET") {
     PrintMsg( 'D', 'process_mode is BAD_DATASET ' );
     copy_log_file( \%group_info, $problem_log_dir, $ds_log_spec );
 }
 # resignal the alarm to the group OSF
 if (!$group_osf && $process_mode eq "ALARM") {
    PrintMsg( 'D', 'group_osf=false AND process_mode is ALARM ' );
    PrintMsg( 'D', 'resignal the alarm to the group OSF ' );
    # reset the status of the group OSF to process the alarm
    PrintMsg("I",
       "Retriggering alarm for group OSF: $group_name $group_data_id");
    my $upd_command="osf_update -p $osf_path -f $group_name".
      " -t $group_data_id -c $update_stage -s $update_alarm";
    if (system($upd_command)) {
        ErrorExit( "Error executing command: $upd_command", $OSF_FAILURE );
    }
 }
 if ($process_mode eq "UPDATE" || 
     ($process_mode eq "BAD_DATASET" && !$group_osf)) {
     PrintMsg( 'D', 'process_mode is UPDATE ' );
     PrintMsg( 'D', 'OR (process_mode is BAD_DATASET AND group_osf=false).' );
     # check if group request is complete 
     PrintMsg( 'D', 'check if group request is complete ' );
     eval { $completed_group = group_completion_status( $db_arch, \%group_info ) } ;
     ErrorExit( $@, $OSF_FAILURE ) if $@ ;
 }
 # signal the completion to the group OSF
 if ($process_mode ne "ALARM" && !$group_osf && $completed_group) {
    PrintMsg( 'D', 'Signal non-group completion to the group OSF. ' );
    PrintMsg( 'D', 'process_mode is either (UPDATE or BAD_DATASET) (i.e. NOT ALARM)' );
    PrintMsg( 'D', 'AND group_osf=false AND completed_group=true ' );
    # reset the status of the group OSF 
    PrintMsg("I",
        "All datasets of the group have ids_install_flag updates.");
    PrintMsg("I",
        "Retriggering group OSF $group_name $group_data_id");
    my $upd_command="osf_update -p $osf_path -f $group_name".
        " -t $group_data_id -c $update_stage -s $update_ready";
    if (system($upd_command)) {
        ErrorExit( "Error executing command: $upd_command", $OSF_FAILURE );
    }
 }
 PrintMsg( 'D', 'Are there any bad datasets in group? ' );
 # check if any bad datasets in group
 eval { $bad_data = is_any_dataset_bad( $db_arch, \%group_info, 
                                 $group_osf, $process_mode, $completed_group ) } ;
 ErrorExit( $@, $OSF_FAILURE ) if $@ ;
 
 # generate database updates if this is group OSF and all installs are OK
 PrintMsg('D', 'Generate database updates IF this is group OSF and all installs are OK ' );
 if ($group_osf && ($process_mode eq "UPDATE") && $completed_group
     && (!$bad_data)) 
 {
     PrintMsg( 'D', 'group_osf=true && completed_group=true && ' );
     PrintMsg( 'D', 'bad_data=false && process_mode is UPDATE ' );
     # check for redundant update
     PrintMsg( 'D', 'check for redundant update ' );
     eval { $needs_insertion = check_archive_table( $db_arch, \%group_info ) } ;
     ErrorExit( $@, $OSF_FAILURE ) if $@ ;

     if ($needs_insertion) {
        PrintMsg( 'D', 'Needs insertion:' );
        # obtain OPUS build number in version variable
        PrintMsg ( 'D', "obtain OPUS build number " );
        eval { $version = get_testbed_version( $sogs_disk ) } ;
        ErrorExit( $@, $OSF_FAILURE ) if $@ ;


##===========================================================================
        # PR 61672   
        # determine proprietary rights information
        # starting with proprietary start time
        PrintMsg( 'I', 'Determining proprietary rights information ' );
        # Default because input could be FUSE, could be a POD, could be a log...
        $prop_start_tm = "'" . $gen_date . "'" ;     ## Quote it for insert
        PrintMsg( 'I', "default prop_start_tm  is $prop_start_tm " );
        if ( index( $proprietary, $archive_class ) > -1 ) {
            $prop_start_tm = check_restricted_data( $db_arch, \%group_info );
            if ( $prop_start_tm eq "" ) { 
                my $msg = "Proprietary start time MUST NOT be blank " ;
                ErrorExit( $msg, $OSF_FAILURE ) ;
            }
            $prop_start_tm = "'" . $prop_start_tm . "'" ; ## Quote it for insert
            PrintMsg( 'D', "prop_start_tm          is $prop_start_tm " );

            # Check ASSIST tables for proprietary entries
            # for use in determining what value, if any, to add to 
            # ads_proprietary_start_time to create ads_release_date
            # but ONLY if rootname is in list of PROPRIETARY classes

            PrintMsg( 'I', "Querying $ASSIST_DB for $pep_id $group_name ($archive_class class) proprietary period " );
            # archive_class DOES indicate proprietary data, 
            # therefore find proprietary period in ASSIST tables, 
            # else email when default period is used

            # ASSIST_DB obs_proprietary_period table: 
            # ALL members affected IF one member found in obs prop per tbl
            # If NO members are found in obs prop per tbl, then a string is returned
            PrintMsg( 'D', "Call check_obs_prop_per( $db_arch, \%group_info, $ds_type_versions, $db_assi, $db_opus )" );
            my $prop_per_mos ;
            eval { $prop_per_mos = check_obs_prop_per( $db_arch, \%group_info, $ds_type_versions, $db_assi, $db_opus ) } ;
            my $msg = "proprietary period uncertain for $group_name ";
            ErrorExit( $msg, $OSF_FAILURE ) if $@ ;
            PrintMsg( 'I', "group $group_name observation proprietary period is $prop_per_mos " );
    
            # ASSIST_DB prog_proprietary_period table: 
            # If check_obs_prop_per found an int above, we will not see next 
            # debug and will NOT execute the following
            unless ( $prop_per_mos =~ /^\d+$/ ) {
                $prop_per_mos = check_prog_prop_per( $db_assi, $pep_id );
               #PrintMsg( 'D', "prog prop_per_mos for $pep_id is $prop_per_mos" );
                PrintMsg( 'I', "proposal $pep_id program proprietary period is $prop_per_mos" );
            }
    
            # else DEFAULT:
            # ONLY execute the send email block IF above 2 checks failed (to get numbers)
            # UNLESS archive_class is NOT expected in above tables (eg. CSI, SSI) PR 65997 
            my $no_email = 0 ;
            if ( index( "SSI,CSI", $archive_class ) > -1 ) {
                $no_email = 1;
            }
            unless ( $prop_per_mos =~ /^\d+$/ ) {
                # No entry in prog_proprietary_period table either
                my $msg = "Proprietary period will DEFAULT to $release_delay_months " ;
                PrintMsg( 'I', $msg );
                $prop_per_mos = $release_delay_months ;
                PrintMsg( 'D', "prop_per_mos is $prop_per_mos, set from release_delay_months $release_delay_months" );
                if ( ! $no_email ) {
                    # Notify operator that DEFAULT period was used
                    my $msg = "Send email to $to_address " ;
                    PrintMsg( 'D', $msg );
                    my $stat = send_email( \%group_info, $prop_per_mos, $to_address, $ingest_log_dir ) ;
                    # Not sure how to get above to fail in order to test this
                    if ( $stat ) {
                        my $msg = "process_mode was $process_mode " ;
                        PrintMsg( 'D', $msg );
                        $msg = "process_mode changed to EMAIL_FAILURE (mark OSF) " ;
                        PrintMsg( 'D', $msg );
                        # could NOT send email; is this cause to mark OSF failed?
                        # remote_pipeline will be considered success
                        $process_mode = "EMAIL_FAILURE" ;
                    }
                    else {
                        PrintMsg( 'I', "Sent default period alert email to $to_address " ) ;
                    }
                }
            }
            PrintMsg( 'D', "prop_per_mos           is $prop_per_mos " );
        
            # reassign to the name we want to pass on
            $release_delay_months = $prop_per_mos ;
            PrintMsg( 'D', "release_delay_months   is $release_delay_months, set from prop_per_mos $prop_per_mos " );
        } else {
            # archive_class does NOT indicate proprietary data, 
            # use current proprietary start time (which should match gen_date)
            $release_delay_months = 0 ;
        }
        # Should these be converted to INFO?
        PrintMsg( 'I', "proprietary_start_time is $prop_start_tm " );
        PrintMsg( 'I', "release_delay_months   is $release_delay_months" );
        PrintMsg( 'D', 'END determine proprietary rights information ' );

##===========================================================================

        # insert records into archive tables
        PrintMsg( 'D', 'insert records into archive tables ' );
        eval { insert_archive_tables( $db_arch, \%group_info, $proprietary, 
                               $override_data_source, $data_source, $version, 
                               $release_delay_months, $prop_start_tm ) } ;
        ErrorExit( $@, $OSF_FAILURE ) if $@ ;
        
        # fill ArchiveFileNsaFileInfo table
        PrintMsg( 'D', 'insert records into ArchiveFileNsaFileInfo table ' );
        eval { insert_afi_nsa_info ( $db_arch, \%group_info ) } ;
        ErrorExit( $@, $OSF_FAILURE ) if $@ ;
        
        # correct ads_best_version values if required
        # PR 64912 Collect qs_split info for set_a_date 
        PrintMsg( 'D', 'correct ads_best_version values if required ' );
        my $qs_split = best_version( $db_arch, \%group_info, $ds_type_versions );

        # PR 64912: pass qs_split
        # use old release dates from old records 
        PrintMsg( 'D', 'Use old release dates from old records? ' );
        set_a_date ( $db_arch, \%group_info, $ds_type_versions, $qs_split, 
                           "ads_release_date", "ads_release_date_mod" );

##===========================================================================
        # PR 61672   (PR 64912: pass qs_split)
        # use old proprietary start dates from old records 
        PrintMsg( 'D', 'Use old proprietary start dates from old records? ' );
        set_a_date ( $db_arch, \%group_info, $ds_type_versions, $qs_split, 
                           "ads_proprietary_start_time" );
##===========================================================================

        # report all archive db data to $trl file only
        PrintMsg( 'D', "report all ads/afi insertions to $ds_log_spec (only)" );
        eval { report_archive_table_fields( $db_arch, \%group_info ) } ;
        ErrorExit( $@, $OSF_FAILURE ) if $@ ;
     }
     else {
        PrintMsg( 'I', 'Nothing else to do. ') ;
     } # end if needs_insertion
 }
 
 # end of all queries
 PrintMsg( 'D', 'end of all queries ' );
#$db_arch->dbclose;
#undef $db_arch;
 
 if ($group_osf && $remote_params{remote_path_file} ne "NONE") {
     # send status to remote pipeline
     eval { send_remote_status( \%group_info, $completed_group, 
                                $process_mode, $needs_insertion, 
                                $bad_data, \%remote_params ) } ;
     ErrorExit( $@, $OSF_FAILURE ) if $@ ;
     my $msg = "...called send_remote_status ( \%group_info, " ;
     $msg = $msg . "$completed_group, $process_mode, $needs_insertion, " ;
     $msg = $msg . "$bad_data, \%remote_params ) " ;
     PrintMsg( 'D', $msg );
 }
 # set exit status at end of processing
 my $final_status = $OSF_SUCCESS;
 PrintMsg( 'D', 'set exit status at end of processing ' );
 if ($process_mode eq "ALARM") {
    PrintMsg("I","Setting OSF status to OSF_DONE after alarm processing.");
    $final_status = $OSF_DONE;
 } elsif ($group_osf && $process_mode eq "UPDATE" && !$completed_group) {
    PrintMsg( "I", "Found ingest_data_set_info.ids_install_flag set to N.");
    PrintMsg( "I", "Setting OSF status to OSF_VAIT.");
    $final_status = $OSF_VAIT;
 } elsif ( $process_mode eq "EMAIL_FAILURE" ) {
    PrintMsg( "E", "Found email failure. " );
    PrintMsg( "I", "Setting OSF status to OSF_FAILURE. " );
    $final_status = $OSF_FAILURE;
 }
 PrintMsg ("I",
      "---  end  --- Ingest Update Archive for $osf_root $osf_data_id ---");
 PrintMsg ( 'D', "exit final_status $final_status " ) ;
#print "## TRACE exit final_status ( $final_status ) \n" ; # in case no debug
 exit( $final_status );

END
{
   PrintMsg( 'I', 'Closing all database connections.' ) ;
   if (defined($db_arch)) {
       DoDBIclose($db_arch);
   }
   if (defined($db_spss)) {
       DoDBIclose($db_spss);
   }
   if (defined($db_assi)) {
       DoDBIclose($db_assi);
   }
   if (defined($db_opus)) {
       DoDBIclose($db_opus);
   }
   undef $db_arch;
   undef $db_spss;
   undef $db_assi;
   undef $db_opus;
}

#----------------------------------------------------------------------------
# Name: ErrorExit 
#       exit the script with the error condition 
#       and printing the error message 
#
# Input:
#       msg     - the error message
#       errCode - the error code to send to OS
#
# Output:
#       txt message
#       error code
#
# History:
# yy/mm/dd PR     Who         Description
# 09/10/29 61672  Sherbert    switch around a bit
#
#----------------------------------------------------------------------------
sub ErrorExit ($$) {   
   # exit the script with the error condition 
   # after closing the database objects 
   # and writing the error message to the log file.
   
   my ($msg, $errCode) = @_;
   PrintMsg("E", $msg);
   exit ( $errCode );
} # end ErrorExit