#!/usr/bin/env perl 
#----------------------------------------------------------------------------
#
# Name: DCF_collect
#
#The basic logic of this script is as follows
#
#    for each association which this exposure is a member {
#	if there is an association OSF {
#	    decrement the OSF count by one
#	} else {
#	    get the count of exposures for this association
#	    decrement the count by one
#	    create an association OSF
#	}
#	if (count == 0) {
#	    update the OSF to complete
#	}
#    }
#
# Usage:
#	%dcn_collect
#	    note that the following environment variables are expected:
#		OSF_DATASET, DSQUERY, OPUS_DB, PATH_BASENAME
#
#	    The resource file keywords ASN_CLASS identifies the OSF
#	    class of the association, and the ASN_STEP identifies
#	    the column (step) in the pipeline which should be set for
#	    the association.
#
#	    If RECOLLECT_OK is set to TRUE, then already-collected ASNs
#	    can be re-collected.
#           If CREATE_ASN_OSF is set to TRUE, then association OSFs will
#           be created as needed.
#	    If WRITE_TRAILER is set to TRUE, messages will be written to
#	    trailer files(For the FGS pipeline which has no trailer files
#	    set WRITE_TRAILER to FALSE(Default is TRUE).
#
# History:
# Date     OPR      Who         Reason
# -------- -------- ----------  ---------------------------------------------
# 06/24/99  38930    Rose       Initial code
# 08/19/99  38930_04 Rose	Quote the stage for OAPI
# 09/01/99  38930_05 Rose	Logic on is_collected is backwards
# 09/13/99  38930_06 Rose	Remove filename separators
# 05/15/00  41303    Heller	exit status 101 is reserved for fatal errors
# 08/21/00  39971    Ken S.	Get trigger values from resource file.
# 03/19/01  43165_06 MSwam      Add OTFR_WORLD and test for it, Perl cleanup
# 03/29/01  43165_10 MSwam      Replace OPUS_TESTING with OTFR_WORLD
# 04/04/01  43165_12 MSwam      Fix dayofyear (yday) in date,add STDB reporting
# 05/08/01  40922    Ken S.     Send osf_test sys$error DC_COLLECT process log.
# 07/05/01  43988    Ken S.     Add WRITE_TRAILER and test for it.
# 06/24/02  46012    J.Baum     For DB_get_count, generalize to count records
#                               with member_status not orphans and not products.
# 08/01/02  45761    MSwam      Treat orphans as singletons for OTFR
# 01/09/03  47164    J.Baum     Process NICMOS trailer depends on WRITE_TRAILER
# 09/29/03  49496    MSwam      Changes for OTFR association handling
#                               (OTFR_WORLD becomes RECOLLECT_OK, and
#                                added CREATE_ASN_OSF flag)
# 08/11/05  53803    MSwam      Replace PATH_FILE_NAME with PATH_BASENAME
# 01/14/09  64274    MSwam      Switch to DBI database access
#----------------------------------------------------------------------------

#---------------------------------------------------------------------
# The following error codes may change in the future
#---------------------------------------------------------------------
unshift @INC,(split /:/, $ENV{PATH});
require 'do_dbi_pkg.pl';

$OSF_MEMBER       =  11;
$OSF_SINGLE	  =  13;
$OSF_DUPLICATE    =  15;
$OTFR_MEMBER_ONLY =  17;
$NO_RECORDS_ERR   =   7;
$OSF_TEST_ERROR   =   7;
$OSF_ALREADY_ZERO =   7;
$OSF_CREATE_ERROR =   7;
$OSF_UPDATE_ERROR =   7;

if (exists($ENV{"WRITE_TRAILER"})) {
   $WRITE_TRAILER  = ($ENV{"WRITE_TRAILER"} =~ /TRUE/);
} else {
   $WRITE_TRAILER = 1;
   print "* * * setting WRITE_TRAILER to default value of $WRITE_TRAILER \n";
}
$RECOLLECT_OK = ($ENV{"RECOLLECT_OK"} =~ /TRUE/);
$CREATE_ASN_OSF = ($ENV{"CREATE_ASN_OSF"} =~ /TRUE/);
$STDB_REPORT_LEVEL = ( $ENV{"STDB_REPORT_LEVEL"} );

print "* * * WRITE_TRAILER is $WRITE_TRAILER \n";
print "* * * RECOLLECT_OK is $RECOLLECT_OK\n";
print "* * * CREATE_ASN_OSF is $CREATE_ASN_OSF\n";
print "* * * STDB_REPORT_LEVEL is $STDB_REPORT_LEVEL \n";
print "* * * DSQUERY is $ENV{'DSQUERY'}, OPUS_DB is $ENV{'OPUS_DB'}\n";

#---------------------------------------------------------------------
# flag the beginning of processing
#---------------------------------------------------------------------
    $banner = "-"x60;
    $expname = $ENV{"OSF_DATASET"};
    $expname = lc($expname);

    $PGM_ID  = uc(substr($expname,1,3));
    $OBS_ID  = uc(substr($expname,4,2));
    $OB_NUM  = uc(substr($expname,6,2));

    $asn_class = $ENV{"ASN_CLASS"};
    $asn_class = lc($asn_class);
    $asn_step  = $ENV{"ASN_STEP"};
    $mem_vait = $ENV{"MEM_VAIT"};
    $mem_vait  = lc($mem_vait);
    $asn_vait = $ENV{"ASN_VAIT"};
    $asn_vait  = lc($asn_vait);

    ($sec,$min,$hour,$ignore_mday,$ignore_mon,$year,$ignore_wday,$yday,$ignore_isdst) = localtime(time);
    $year   += 1900;
    $yday   += 1;
    $date = sprintf("%4d%03d%02d%02d%02d",$year,$yday,$hour,$min,$sec);
    $path    = $ENV{"PATH_BASENAME"};
    $colon   = rindex($path,":");
    $path    = substr($path,$colon+1);
    $caldir  = `osfile_stretch_file $ENV{"OUTPATH"}`;
    $sisdir  = `osfile_stretch_file $ENV{"INPATH"}`;
    chop($caldir);
    chop($sisdir);
    print "$date-I---------------- Data Collector: $expname ------------------\n";

#---------------------------------------------------------------------
# get the list of associations for this exposure
#---------------------------------------------------------------------
    $count = &DB_get_associations;
    if ($count == 0) {
    	print "$date-I-No records found in the database for $expname\n";
	print "$date-I-Assuming $expname is a singleton or orphan\n";
    	exit ($OSF_SINGLE);
    } elsif ($count < 0) {
	print "$date-E-DUP $expname is already collected\n";
	exit ($OSF_DUPLICATE);
    }

    $asn_count = 0;
    foreach $asn (@associations) {
	$lasn = lc($asn);

	$is_collected = &DB_is_collected;
	if ($is_collected) {
	    print "$date-I-DUP $lasn is already collected\n";
	    print "$date-I-Assuming $expname is a singleton\n";
	    exit ($OSF_SINGLE);
  	}

#---------------------------------------------------------------------
#   	determine if there is an association OSF and its dcf_number
#---------------------------------------------------------------------
	$cmd = "osf_test -p $path -f $lasn -pr dcf_num";
	print "$date-I-$cmd\n";
    	$number = `$cmd`;
        if ($?) {
	    print "$date-E-osf_test failed for $lasn \n";
            print "$number";
	    exit ($OSF_TEST_ERROR);
    	}

    	$trailer = $caldir.$lasn.".tra";

#---------------------------------------------------------------------
#   	if there is no asn OSF, then create one (unless mode says no) 
#---------------------------------------------------------------------
    	if ($number ne "") {
            $number += 0; 	#silly conversion of string to number
            print "$date-I-OSF found for $asn with number=$number\n";
            $asn_osf_found = 1;
            $asn_count = $asn_count + 1;
        } else {
       	    print "$date-W-osf_test unable to find association OSF for $asn\n";
            $asn_osf_found = 0;

#---------------------------------------------------------------------
#	    count the number of exposures in this association
# 	    if there are none in the database, this is a problem
#---------------------------------------------------------------------
    	    $count = &DB_get_count;
    	    if ($count == 0) {
    		print "$date-E-No records found in the database for $asn\n";
    		exit ($NO_RECORDS_ERR);
    	    }

            $number = $count;
            #
            # only create the asn OSF if the mode says to
	    if ($CREATE_ASN_OSF) {
              $numstr = sprintf("%03.3d", $number);
              $cmd = "osf_create -p $path -f $lasn -n $numstr -t $asn_class -c \"$asn_step\" -s $mem_vait";
              print "$date-I-$cmd\n";  
              `$cmd`;
              if ($?) {
   	    	print "$date-E-unable to create the osf for $asn\n";
	     	exit ($OSF_CREATE_ERROR);
              } else {
                # asn osf now created
                $asn_osf_found = 1;
                $asn_count = $asn_count + 1;
	        if ($WRITE_TRAILER) {
	    	   open (TRAILER, ">>".$trailer);
	    	   $msg = "$date-I-created the osf for $asn\n";
	    	   print $msg; print TRAILER $msg;
	    	   close (TRAILER);
	        }
	      }
            }
        }
	$number--;

#---------------------------------------------------------------------
#   	if this is a NICMOS, append the trailer to the product trailer
#---------------------------------------------------------------------

    	if (($expname =~ /^[Nn]/) && $WRITE_TRAILER) {
    	    $fname = $caldir.$expname."_raw.fits";
	    print "$date-I-FNAME: $caldir $expname $fname\n";
    	    $mtype = `kval $fname ASN_MTYP`;
    	    print "$date-I-MTYPE: $expname has MTYPE = $mtype\n";

	    if ($mtype =~ /EXP-TARG/) {
	    	$product = $lasn;
	    } else {  #product = IPPPSSAAN 
	    	$product = lc(substr($lasn,0,8).substr($mtype,7,1));
	    }

	    $prodtrl = $caldir.$product . ".tra";
            #
            # only append to the product trailer if the mode says to
            # or if we are, only if it already exists
            #
            if ($CREATE_ASN_OSF || -e $prodtrl) {
	      $exptrl  = $caldir.$expname . ".tra";
	      print "$date-I-APPND: Appending $exptrl to $prodtrl\n";

              if ($WRITE_TRAILER) {
	        open (TRAILER, ">>".$prodtrl) or die "Open failed on $prodtrl: $!\n";
	        print TRAILER "$banner\n";
	        print TRAILER "------------------------ $expname -------------------------\n";
	        print TRAILER "$banner\n";
	        open (EXPTRAIL, "<".$exptrl) or die "Couldn't open $exptrl: $!\n";
	        while (<EXPTRAIL>) {
	  	   print TRAILER $_;
	        }
	        close (TRAILER);
	      }
            }
	} elsif ($asn_osf_found && $WRITE_TRAILER) { 
            #
            # 'touch' the association trailer
	    open (TRAILER, ">>".$sisdir.$lasn.".trx");
	    close (TRAILER);
    	}
    
#---------------------------------------------------------------------
#    	if the member counter is down to zero then 
#       send the observation on to next step
#       (unless the mode says not to and no asn OSF exists in which
#        case no action is needed)
#---------------------------------------------------------------------
        if ($CREATE_ASN_OSF || $asn_osf_found == 1) {
    	  $numstr = sprintf("%03.3d", $number);
    	  $oldnum = sprintf("%03.3d", $number+1);
    	  if ($number == 0) {
	    print "$date-I-\$number = :$number:\n";
  	    $cmd = "osf_update -p $path -f $lasn -n 001 -m 000 -c \"$asn_step\" -s $asn_vait";
            print "$date-I-$cmd\n";  
            `$cmd`;
	    print "$date-I-status = $?\n";
    	  } elsif ($number < 0) {
            # woops, the member counter has counted down too far
	    if (! $CREATE_ASN_OSF) {
	       print "$date-W-Shared exposure collision? $asn already has number=0\n";
	       print "$date-W-Skipping it.\n";
               next;  # ignore this for OTFR
	    }
	    else {
	       print "$date-E-YIKES $asn already has number=0\n";
               exit ($OSF_ALREADY_ZERO);
            }
    	  } else {
  	    $cmd="osf_update -p $path -f $lasn -n $oldnum -m $numstr -c \"$asn_step\" -s $mem_vait";
            print "$date-I-$cmd\n";  
            `$cmd`;
      	  }
    	  if ($?) {
	    print "$date-E-OSF_update failed for $asn\n";
	    exit ($OSF_UPDATE_ERROR);
    	  }

          # update the trailer with the exposure name
          $msg = "$date-I---------------- Collected: $expname ------------------\n";
          if ($WRITE_TRAILER) {
            open (TRAILER, ">>".$trailer);
            print $msg; print TRAILER $msg;
            close (TRAILER);
          }
        } #end $CREATE_ASN_OSF or $asn_osf_found == 1
    } #end foreach

#---------------------------------------------------------------------
#   set exit to obtain proper triggering of next stage
#---------------------------------------------------------------------
    #
    # add an exit status for an OTFR member-only request
    # (known if we reach here in the right mode without any asn OSF found)
    #
    if (! $CREATE_ASN_OSF && $asn_count == 0) {
      exit ($OTFR_MEMBER_ONLY);
    }
    else {
      exit ($OSF_MEMBER);
    }


#############################################################################
# 
# Name: DB_get_associations
#
# Purpose: Returns a list of all ASNs for which an exposure (as defined by
#          globals PGM_ID, OBS_ID, and OB_NUM) is a member.  If not in
#          RECOLLECT_OK mode, it must be an uncollected member.  Orphans
#          should not be treated as association members.
#
# Globals in: 
#          PGM_ID
#          OBS_ID
#          OB_NUM
#          DSQUERY
#          OPUS_DB
#          RECOLLECT_OK
#          date
#          expname
# Globals out:
#          associations() - list of association_ids 
#
##############################################################################
sub DB_get_associations {

    my ($db, $query, $sth, $count, $association, $member_status);  # LOCAL VARS
	 
    my $EXIT_FAILURE = 0;
    $db = DoDBIopen( $ENV{"DSQUERY"}, lc($ENV{"OPUS_DB"}), $EXIT_FAILURE);

#---------------------------------------------------------------------
# set up the query
#---------------------------------------------------------------------
$query = <<"EOQ";
SELECT association_id,member_status FROM asn_members
WHERE program_id = '$PGM_ID'
AND   obset_id   = '$OBS_ID'
AND   member_num = '$OB_NUM'
EOQ

if ($STDB_REPORT_LEVEL) { 
    print "\n* * * Now in DB_get_associations subroutine... \n" ; 
    print "\n",$query,"\n" ;
}
@associations = ();
print "$date-I-expname=$expname, $PGM_ID, $OBS_ID, $OB_NUM\n";
	 
#---------------------------------------------------------------------
# count the number of associations for this exposure
#---------------------------------------------------------------------
    $count = 0;	 
    my $sth = DoDBIexecute( $db, $query);

    while ( my ($association, $member_status) = DoDBIfetch($db, $query, $sth)) {
	if (!$RECOLLECT_OK && $member_status =~ /C/) {
	    $count=-999;
	}
        # orphans are NOT association members
	if ($member_status =~ /O/) {
	    next;
	}
	print "$date-I-\$association = :$association:\n";	#for testing
	push(@associations, $association);
	$count++;
    }
    print "$date-I-count = $count\n";	 
    $sth->finish;
    DoDBIclose($db);
    $count;	# return value

}


#############################################################################
#
# Name: DB_get_count
#
# Purpose: Returns a count of the association members for a particular
#          asn.  Always count non-products that are also non-orphans. This
#          works for RECOLLECT_OK both true and false.
#
# Globals in: DSQUERY
#             OPUS_DB
#             asn
#  
# Returns: count of members in the asn
#          
##############################################################################
sub DB_get_count {
    my ($db, $query, $sth, $s, $count, $k, $v, %info);   # LOCAL VARS 
	 
    my $EXIT_FAILURE = 0;
    $db = DoDBIopen( $ENV{"DSQUERY"}, lc($ENV{"OPUS_DB"}), $EXIT_FAILURE);

$query = <<"EOQ";
SELECT * from asn_members
WHERE association_id = '$asn'
AND member_status != 'O'
AND member_status != 'P'		
EOQ
	 
    if ($STDB_REPORT_LEVEL) {
        print "\n* * * Now in DB_get_count subroutine... \n" ;
        print "\n",$query,"\n" ;
    }

    $count = 0;	 
    my $sth = DoDBIexecute( $db, $query);

    while ( (%info) = DoDBIfetch($db, $query, $sth)) {
	print "$date-I-found: ";
	while ( ($k,$v) = each %info ) {
	   print "$v\t";
	}
	print "\n";
	$count++;
    }
    print "$date-I-count = $count\n";	 
    $sth->finish;
    DoDBIclose($db);
    $count;
}
#############################################################################
#
# Name: DB_is_collected
#
# Purpose: Identifies if a particular ASN has already been collected or
#          not by examining the asn_association.collect_date db field.
#
# Globals in: DSQUERY
#             OPUS_DB
#             asn
#             RECOLLECT_OK
#
# Return:  1 if ASN was already collected and RECOLLECT_OK = FALSE
#          0 otherwise  (always = 0 if RECOLLECT_OK = TRUE)
#
##############################################################################
sub DB_is_collected {
    my ($db, $query, $sth, $count, $collect_date); #LOCAL VARS
	 
    my $EXIT_FAILURE = 0;
    $db = DoDBIopen( $ENV{"DSQUERY"}, lc($ENV{"OPUS_DB"}), $EXIT_FAILURE);

$query = <<"EOQ";
SELECT collect_date from asn_association
WHERE association_id = '$asn'
EOQ
	 
    if ($STDB_REPORT_LEVEL) {
        print "\n* * * Now in DB_is_collected subroutine... \n" ;
        print "\n",$query,"\n" ;
    }

    $count = 0;	 
    my $sth = DoDBIexecute( $db, $query);
    $collect_date = DoDBIfetch($db, $query, $sth);

    if ($collect_date =~ /:/) {
	$count = 1;
    } else {
	$count = 0;
    }
    $sth->finish;
    DoDBIclose($db);

    #override this if RECOLLECT_OK
    if ($RECOLLECT_OK) {
	$count=0;
    }

    $count;
}