#!/bin/csh
##########################################################################
#
# routine: ingxfer.csh
#
# purpose: Script for transfering science data to be ingested.
#
# modification history:
#
#   date    opr     who     reason
# -------- -----  --------  --------------------------------------
# 07/11/12 70739  FHam      first version
# 12/17/12 73112  FHam      Add new daily directory functionality
# 01/18/13 73112  FHam      Fixed asn product no transfer bug
##########################################################################
set banner=">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
echo $banner
/bin/date

#-------------------------------------------------------
# set exit status values
#-------------------------------------------------------

set go_absent = 102          # causes XPOLL to error exit
set all_is_well = 0          # nominal completion
set quit_this_dataset = 1    # status mapped to OSF_ERROR
set duplicate_osf = 2        # status set for duplicate OSF in remote ingest pipe

#-------------------------------------------------------
# Check to see if the remote OSF bb is up before we do anything.
#-------------------------------------------------------

set bb_count = `ssh $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE ps -ef | grep $INGEST_REMOTE_ACCOUNT | grep opus_bb_server | grep OSF | grep $INGEST_REMOTE_PATH | wc -l`

if ($bb_count != 1) then
    echo "The OSF BB server for $INGEST_REMOTE_PATH on $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE is not up!"
    exit $go_absent
endif

#-------------------------------------------------------
# check for duplicate OSF in remote ingest pipe
#-------------------------------------------------------

set osf_list = `ssh $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE osf_test -p $INGEST_REMOTE_PATH -f $OSF_DATASET | wc -l`
if ($status) then
   echo "Cannot access pipeline $INGEST_REMOTE_PATH using osf_test."
   echo "Please verify pipeline servers are active and try again."
   exit $go_absent
endif 

if ($osf_list != 0) then
   echo "Duplicate OSF for $OSF_DATASET found in path $INGEST_REMOTE_PATH on $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE."
   exit $duplicate_osf
endif   

#-------------------------------------------------------
# determine if this is an ASN
#-------------------------------------------------------

set lastchar = `echo $OSF_DATASET | /bin/awk '{a=substr($1,length,1); print a}'`

if ($lastchar =~ [0-9a-i] && $OSF_DATA_ID != 'ssi' && $OSF_DATA_ID != 'csi') then
   set is_asn = 1
else
   set is_asn = 0
endif

#-------------------------------------------------------
# define directory where the data live
#-------------------------------------------------------

if ($OSF_DATA_ID != 'ssi' && $OSF_DATA_ID != 'csi') then
   set datadir = `osfile_stretch_file $INPATH`
else
   set datadir = `osfile_stretch_file $MAMA_DIR`
endif

#-------------------------------------------------------
# move to working directory
#-------------------------------------------------------

if (! -e $datadir) then
   echo "ERROR: $datadir directory does not exist"
   echo $banner   
   exit $go_absent
endif
cd $datadir
if ($status != 0) then
   echo "ERROR: Failed to set directory to $datadir"
   echo $banner
   exit $go_absent
endif

#-------------------------------------------------------
# if local alt dir copy needed, create directory
#-------------------------------------------------------

if ($ALT_LOCAL_DAILY_DIR != 'none') then
   set day_of_year_date = ( eval "date '+%Y%j_%a_%b%d'" )
   set local_daily_dir = ${ALT_LOCAL_DAILY_DIR}/"`${day_of_year_date}`"

   # the local daily directory does not exist, create it

   if (! -e ${local_daily_dir}) then
      mkdir $local_daily_dir
      if ($status != 0) then
         echo "ERROR: Failed to create local daily directory $local_daily_dir."
         exit $quit_this_dataset
      endif
   endif
endif

#-------------------------------------------------------
# transfer the data over to the ingest machine and 
# alternate site
#-------------------------------------------------------

if ($is_asn) then
   echo "Transfering ASN $OSF_DATASET to ${INGEST_REMOTE_MACHINE}:${INGEST_REMOTE_DIR}."
   set members = `listasn ${OSF_DATASET}_asn.fits -s | /bin/awk '{print $1}' | /bin/dd conv=lcase`

   ## splitting long line above with continuation chars causing an error

   if ($#members == 0) then
      echo "WARNING: Failed to generate list of members for ${OSF_DATASET}: no trailers copied"
   else

      # transfer ASN files

      echo "Transfering $OSF_DATASET to ${INGEST_REMOTE_MACHINE}:${INGEST_REMOTE_DIR}."

      scp $OSF_DATASET* ${INGEST_REMOTE_ACCOUNT}@${INGEST_REMOTE_MACHINE}:${INGEST_REMOTE_DIR}
      if ($status != 0) then
         echo "scp of $OSF_DATASET to ${INGEST_REMOTE_MACHINE}${INGEST_REMOTE_DIR} failed."
         exit $quit_this_dataset
      endif

      # transfer files to alternate directory mounted on the local machine, 
      # if ALT_LOCAL_DAILY_DIR variable is != none 

      if ($ALT_LOCAL_DAILY_DIR != 'none') then
         
         # transfer ASN files to the local daily directory

         /bin/cp $OSF_DATASET* ${local_daily_dir}
         if ($status != 0) then
            echo "cp of $OSF_DATASET to local daily directory ${local_daily_dir} failed."
            exit $quit_this_dataset
         endif
      endif

      # transfer files to alternate site, if ALT variables are defined

      if ($ALT_REMOTE_ACCOUNT != 'none') then
         rsync -z $OSF_DATASET* ${ALT_REMOTE_ACCOUNT}@${ALT_REMOTE_MACHINE}:${ALT_REMOTE_DIR}
         if ($status != 0) then
            echo "rsync of $OSF_DATASET to ${ALT_REMOTE_MACHINE}${ALT_REMOTE_DIR} failed."
            exit $quit_this_dataset
         endif
      endif

      # transfer ASN members, except for STIS which does not ingest members

      if ($OSF_DATA_ID != 'sas') then
         foreach MEMBER_ROOT ( $members ) 
            set num_member_files = `ls ${MEMBER_ROOT}* | wc -l`
            echo "transfering ${num_member_files} files for member ${MEMBER_ROOT}"
            if ($num_member_files != 0) then
               scp $MEMBER_ROOT* ${INGEST_REMOTE_ACCOUNT}@${INGEST_REMOTE_MACHINE}:${INGEST_REMOTE_DIR}
               if ($status != 0) then
                  echo "scp of $MEMBER_ROOT to ${INGEST_REMOTE_MACHINE}${INGEST_REMOTE_DIR} failed."
                  exit $quit_this_dataset
               endif

               # transfer files to alternate directory mounted on the local machine,
               # if ALT_LOCAL_DAILY_DIR variable is != none

               if ($ALT_LOCAL_DAILY_DIR != 'none') then

                  # transfer ASN files to the local daily directory

                  /bin/cp $MEMBER_ROOT* ${local_daily_dir}
                  if ($status != 0) then
                     echo "cp of $MEMBER_ROOT to local daily directory ${local_daily_dir} failed."
                     exit $quit_this_dataset
                  endif
               endif

               # transfer files to alternate site, if ALT variables are defined

               if ($ALT_REMOTE_ACCOUNT != 'none') then
                  rsync -z $MEMBER_ROOT* ${ALT_REMOTE_ACCOUNT}@${ALT_REMOTE_MACHINE}:${ALT_REMOTE_DIR}
                  if ($status != 0) then
                     echo "rsync of $MEMBER_ROOT to ${ALT_REMOTE_MACHINE}${ALT_REMOTE_DIR} failed."
                     exit $quit_this_dataset
                  endif
               endif
            else
               echo "$MEMBER_ROOT not found, thus cannot transfer it to ${INGEST_REMOTE_MACHINE}${INGEST_REMOTE_DIR}."
               continue
            endif
         end
      endif
   endif
else
   echo "Transfering $OSF_DATASET to ${INGEST_REMOTE_MACHINE}:${INGEST_REMOTE_DIR}."

   #
   # Weed out extraneous csum temporary files
   #

   set csum_tmp_file = ${OSF_DATASET}_rcsum_tmp.fits

   if (-e $csum_tmp_file) then
      rm $csum_tmp_file
   endif

   scp $OSF_DATASET* ${INGEST_REMOTE_ACCOUNT}@${INGEST_REMOTE_MACHINE}:${INGEST_REMOTE_DIR}
   if ($status != 0) then
       echo "scp of $OSF_DATASET to $INGEST_REMOTE_MACHINE}${INGEST_REMOTE_DIR} failed."
       exit $quit_this_dataset
   endif

   # transfer files to alternate directory mounted on the local machine,
   # if ALT_LOCAL_DAILY_DIR variable is != none

   if ($ALT_LOCAL_DAILY_DIR != 'none') then

      # transfer singelton files to the local daily directory

      /bin/cp $OSF_DATASET* ${local_daily_dir}
      if ($status != 0) then
         echo "cp of $OSF_DATASET to local daily directory ${local_daily_dir} failed."
         exit $quit_this_dataset
      endif
   endif

   # transfer files to alternate site, if ALT variables are defined

   if ($ALT_REMOTE_ACCOUNT != 'none') then
      rsync -z $OSF_DATASET* ${ALT_REMOTE_ACCOUNT}@${ALT_REMOTE_MACHINE}:${ALT_REMOTE_DIR}
      if ($status != 0) then
         echo "rsync of $OSF_DATASET to ${ALT_REMOTE_MACHINE}${ALT_REMOTE_DIR} failed."
         exit $quit_this_dataset
      endif
   endif

endif

#-------------------------------------------------------
# now create an OSF on the remote machine in the 
# appropriate path
#-------------------------------------------------------

echo "Creating OSF for $OSF_DATASET in the $INGEST_REMOTE_PATH path on $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE."
ssh $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE osf_create -p $INGEST_REMOTE_PATH -f $OSF_DATASET -t $OSF_DATA_ID -c $INGEST_REMOTE_STAGE -s $INGEST_REMOTE_VALUE
if ($status != 0) then
   echo "Creation of OSF for $OSF_DATASET in the $INGEST_REMOTE_PATH path on $INGEST_REMOTE_ACCOUNT@$INGEST_REMOTE_MACHINE failed."
   exit $quit_this_dataset
endif

exit $all_is_well
