"""
Process input to MultiDrizzle/PyDrizzle.

:Authors: Warren Hack

:License: :doc:`LICENSE`

The input can be one of:

    * a python list of files
    * a comma separated string of filenames (including wild card characters)
    * an association table
    * an @file (can have a second column with names of ivm files)

No mixture of instruments is allowed.
No mixture of association tables, @files and regular fits files is allowed.
Files can be in GEIS or MEF format (but not waiver fits).

Runs some sanity checks on the input files.
If necessary converts files to MEF format (this should not be left to `makewcs`
because `updatewcs` may be `False`\ ).
Runs makewcs.
The function `process_input` returns an association table, ivmlist, output name

The common interface interpreter for MultiDrizzle tasks, 'processCommonInput()',
not only runs 'process_input()' but 'createImageObject()' and 'defineOutput()'
as well to fully setup all inputs for use with the rest of the MultiDrizzle
steps either as stand-alone tasks or internally to MultiDrizzle itself.

"""

from __future__ import absolute_import, division, print_function  # confidence high
import datetime
import os
import shutil
import string
import sys

import numpy as np
import astropy
from astropy.io import fits

from stwcs import updatewcs as uw
from stwcs.wcsutil import altwcs, wcscorr
from stsci.tools import (cfgpars, parseinput, fileutil, asnutil, irafglob,
                         check_files, logutil, mputil, textutil)
try:
    from stsci.tools.bitmask import interpret_bit_flags
except ImportError:
    from stsci.tools.bitmask import interpret_bits_value as interpret_bit_flags


from . import wcs_functions
from . import util
from . import resetbits
from . import mdzhandler

log = logutil.create_logger(__name__, level=logutil.logging.NOTSET)

# list parameters which correspond to steps where multiprocessing can be used
parallel_steps = [(3,'driz_separate'),(6,'driz_cr')]

if util.can_parallel:
    import multiprocessing


def setCommonInput(configObj, createOutwcs=True):
    """
    The common interface interpreter for MultiDrizzle tasks which not only runs
    'process_input()' but 'createImageObject()' and 'defineOutput()' as well to
    fully setup all inputs for use with the rest of the MultiDrizzle steps either
    as stand-alone tasks or internally to MultiDrizzle itself.

    Parameters
    ----------
    configObj : object
        configObj instance or simple dictionary of input parameters
    imageObjectList : list of imageObject objects
        list of imageObject instances, 1 for each input exposure
    outwcs : object
        imageObject instance defining the final output frame

    Notes
    -----
    At a minimum, the configObj instance (dictionary) should contain:
        configObj = {'input':None,'output':None }

    If provided, the configObj should contain the values of all the multidrizzle parameters
    as set by the user with TEAL. If no configObj is given, it will retrieve
    the default values automatically.  In either case, the values from the input_dict
    will be merged in with the configObj before being used by the rest of the
    code.

    Examples
    --------
    You can set *createOutwcs=False* for the cases where you only want the
    images processed and no output wcs information in necessary; as in:

    >>>imageObjectList,outwcs = processInput.processCommonInput(configObj)


    """
    # make sure 'updatewcs' is set to False when running from GUI or if missing
    # from configObj:
    if 'updatewcs' not in configObj:
        configObj['updatewcs'] = False

    if not createOutwcs or not configObj['coeffs']:
        # we're probably just working on single images here
        configObj['updatewcs']=False

    # maybe we can chunk this part up some more so that we can call just the
    # parts we want

    # Interpret input, read and convert and update input files, then return
    # list of input filenames and derived output filename
    asndict, ivmlist, output = process_input(
            configObj['input'], configObj['output'],
            updatewcs=configObj['updatewcs'], wcskey=configObj['wcskey'],
            **configObj['STATE OF INPUT FILES'])

    if not asndict:
        return None, None
    # convert the filenames from asndict into a list of full filenames
    files = [fileutil.buildRootname(f) for f in asndict['order']]
    original_files = asndict['original_file_names']

    # interpret MDRIZTAB, if specified, and update configObj accordingly
    # This can be done here because MDRIZTAB does not include values for
    # input, output, or updatewcs.
    if 'mdriztab' in configObj and configObj['mdriztab']:
        print("Reading in MDRIZTAB parameters for {} files".format(len(files)))
        mdriztab_dict = mdzhandler.getMdriztabParameters(files)

        # Update configObj with values from mpars
        cfgpars.mergeConfigObj(configObj, mdriztab_dict)

    # Convert interpreted list of input files from process_input into a list
    # of imageObject instances for use by the MultiDrizzle tasks.
    instrpars = configObj['INSTRUMENT PARAMETERS']
    # pass in 'proc_unit' to initialize unit conversions as necessary
    instrpars['proc_unit'] = configObj['proc_unit']

    undistort = True
    if not configObj['coeffs']:
        undistort = False

    # determine whether parallel processing will be performed
    use_parallel = False
    if util.can_parallel:
        # look to see whether steps which can be run using multiprocessing
        # have been turned on
        for stepnum in parallel_steps:
            sname = util.getSectionName(configObj,stepnum[0])
            if configObj[sname][stepnum[1]]:
                use_parallel = True
                break

    # interpret all 'bits' related parameters and convert them to integers
    configObj['resetbits'] = interpret_bit_flags(configObj['resetbits'])
    step3name = util.getSectionName(configObj,3)
    configObj[step3name]['driz_sep_bits'] = interpret_bit_flags(
                                        configObj[step3name]['driz_sep_bits']
    )
    step7name = util.getSectionName(configObj,7)
    configObj[step7name]['final_bits'] = interpret_bit_flags(
                                        configObj[step7name]['final_bits']
    )

    # Verify any refimage parameters to be used
    step3aname = util.getSectionName(configObj,'3a')
    if not util.verifyRefimage(configObj[step3aname]['driz_sep_refimage']):
        msg = 'No refimage with WCS found!\n '+\
        ' This could be caused by one of 2 problems:\n'+\
        '   * filename does not specify an extension with a valid WCS.\n'+\
        '   * can not find the file.\n'+\
        'Please check the filename specified in the "refimage" parameter.'
        print(textutil.textbox(msg))
        return None,None
    step7aname = util.getSectionName(configObj,'7a')
    if not util.verifyRefimage(configObj[step7aname]['final_refimage']):
        msg = 'No refimage with WCS found!\n '+\
        ' This could be caused by one of 2 problems:\n'+\
        '   * filename does not specify an extension with a valid WCS.\n'+\
        '   * can not find the file.\n'+\
        'Please check the filename specified in the "refimage" parameter.'
        print(textutil.textbox(msg))
        return None,None


    # Build imageObject list for all the valid, shift-updated input files
    log.info('-Creating imageObject List as input for processing steps.')
    if 'in_memory' in configObj:
        virtual = configObj['in_memory']
    else:
        virtual = False

    imageObjectList = createImageObjectList(files, instrpars,
                                            group=configObj['group'],
                                            undistort=undistort,
                                            inmemory=virtual)

    # Add original file names as "hidden" attributes of imageObject
    assert(len(original_files) == len(imageObjectList)) #TODO: remove after extensive testing
    for i in range(len(imageObjectList)):
        imageObjectList[i]._original_file_name = original_files[i]

    # apply context parameter
    applyContextPar(imageObjectList, configObj['context'])

    # reset DQ bits if requested by user
    resetDQBits(imageObjectList, cr_bits_value=configObj['resetbits'])

    # Add info about input IVM files at this point to the imageObjectList
    addIVMInputs(imageObjectList, ivmlist)

    if createOutwcs:
        log.info('-Creating output WCS.')

        # Build output WCS and update imageObjectList with output WCS info
        outwcs = wcs_functions.make_outputwcs(imageObjectList, output,
                                              configObj=configObj, perfect=True)
        outwcs.final_wcs.printwcs()
    else:
        outwcs = None

    try:
        # Provide user with some information on resource usage for this run
        # raises ValueError Exception in interactive mode and user quits
        num_cores = configObj.get('num_cores') if use_parallel else 1

        reportResourceUsage(imageObjectList, outwcs, num_cores)
    except ValueError:
        imageObjectList = None

    return imageObjectList, outwcs


def reportResourceUsage(imageObjectList, outwcs, num_cores,
                        interactive=False):
    """ Provide some information to the user on the estimated resource
    usage (primarily memory) for this run.
    """

    from . import imageObject
    if outwcs is None:
        output_mem = 0
    else:
        if isinstance(outwcs,imageObject.WCSObject):
            owcs = outwcs.final_wcs
        else:
            owcs = outwcs
        output_mem = owcs._naxis1*owcs._naxis2*4*3 # bytes used for output arrays
    img1 = imageObjectList[0]
    numchips = 0
    input_mem = 0
    for img in imageObjectList:
        numchips += img._nmembers # account for group parameter set by user

    # if we have the cpus and s/w, ok, but still allow user to set pool size
    pool_size = util.get_pool_size(num_cores, None)
    pool_size = pool_size if (numchips >= pool_size) else numchips

    inimg = 0
    chip_mem = 0
    for img in imageObjectList:
        for chip in range(1,img._numchips+1):
            cmem = img[chip].shape[0]*img[chip].shape[1]*4
            inimg += 1
            if inimg < pool_size:
                input_mem += cmem*2
            if chip_mem == 0:
                chip_mem = cmem
    max_mem = (input_mem + output_mem*pool_size + chip_mem*2)//(1024*1024)

    print('*'*80)
    print('*')
    print('*  Estimated memory usage:  up to %d Mb.'%(max_mem))
    print('*  Output image size:       %d X %d pixels. '%(owcs._naxis1,owcs._naxis2))
    print('*  Output image file:       ~ %d Mb. '%(output_mem//(1024*1024)))
    print('*  Cores available:         %d'%(pool_size))
    print('*')
    print('*'*80)

    if interactive:
        print('Continue with processing?')
        while True:
            if sys.version_info[0] >= 3:
                k = input("(y)es or (n)o").strip()[0].lower()
            else:
                k = raw_input("(y)es or (n)o").strip()[0].lower()

            if k not in ['n', 'y']:
                continue

            if k == 'n':
                raise KeyboardInterrupt("Execution aborted")


def getMdriztabPars(input):
    """ High-level function for getting the parameters from MDRIZTAB

    Used primarily for TEAL interface.
    """
    filelist,output,ivmlist,oldasndict=processFilenames(input,None)

    try:
        mdrizdict = mdzhandler.getMdriztabParameters(filelist)
    except KeyError:
        print('No MDRIZTAB found for "%s". Parameters remain unchanged.'%(filelist[0]))
        mdrizdict = {}

    return mdrizdict

def addIVMInputs(imageObjectList,ivmlist):
    """ Add IVM filenames provided by user to outputNames dictionary for each input imageObject.
    """
    if ivmlist is None:
        return

    for img,ivmname in zip(imageObjectList,ivmlist):
        img.updateIVMName(ivmname)

def checkMultipleFiles(input):
    """ Evaluates the input to determine whether there is 1 or more than 1 valid input file.
    """
    f,i,o,a=buildFileList(input)
    return len(f) > 1

def createImageObjectList(files,instrpars,group=None,
                            undistort=True, inmemory=False):
    """ Returns a list of imageObject instances, 1 for each input image in the list of input filenames.
    """
    imageObjList = []
    mtflag = False
    mt_refimg = None
    for img in files:
        image = _getInputImage(img,group=group)
        image.setInstrumentParameters(instrpars)
        image.compute_wcslin(undistort=undistort)
        if 'MTFLAG' in image._image['PRIMARY'].header:
            # check to see whether we are dealing with moving target observations...
            _keyval = image._image['PRIMARY'].header['MTFLAG']
            if not util.is_blank(_keyval):
                if isinstance(_keyval,bool):
                    mtflag = _keyval
                else:
                    if 'T' in _keyval:
                        mtflag = True
                    else:
                        mtflag = False
        else:
            mtflag = False

        if mtflag:
            print("#####\nProcessing Moving Target Observations using reference image as WCS for all inputs!\n#####\n")
            if mt_refimg is None:
                mt_refimg = image
            else:
                image.set_mt_wcs(mt_refimg)
        image.inmemory = inmemory # set flag for inmemory processing
        # Now add (possibly updated) image object to list
        imageObjList.append(image)
    return imageObjList

def applyContextPar(imageObjectList,contextpar):
    """ Apply the value of the parameter `context` to the input list, setting
        the name of the output context image to None if `context` is False
    """
    for img in imageObjectList:
        img.updateContextImage(contextpar)

def _getInputImage (input,group=None):
    """ Factory function to return appropriate imageObject class instance"""
    # extract primary header and SCI,1 header from input image
    sci_ext = 'SCI'
    if group in [None,'']:
        exten = '[sci,1]'
        phdu = fits.getheader(input, memmap=False)
    else:
        # change to use fits more directly here?
        if group.find(',') > 0:
            grp = group.split(',')
            if grp[0].isalpha():
                grp = (grp[0],int(grp[1]))
            else:
                grp = int(grp[0])
        else:
            grp = int(group)
        phdu = fits.getheader(input, memmap=False)
        phdu.extend(fits.getheader(input, ext=grp, memmap=False))

    # Extract the instrument name for the data that is being processed by Multidrizzle
    _instrument = phdu['INSTRUME']

    # Determine the instrument detector in use.  NICMOS is a special case because it does
    # not use the 'DETECTOR' keyword.  It instead used 'CAMERA' to identify which of it's
    # 3 camera's is in use.  All other instruments support the 'DETECTOR' keyword.
    if _instrument == 'NICMOS':
        _detector = phdu['CAMERA']
    else:
        try:
            _detector = phdu['DETECTOR']
        except KeyError:
            # using the phdu as set above (fits.getheader) is MUCH faster and
            # works for the majority of data; but fileutil handles waivered fits
            phdu = fileutil.getHeader(input+exten)
            _detector = phdu['DETECTOR'] # if this fails, let it throw

    del phdu # just to keep clean
    # Match up the instrument and detector with the right class
    # only importing the instrument modules as needed.
    try:
        if _instrument == 'ACS':
            from . import acsData
            if _detector == 'HRC': return acsData.HRCInputImage(input,group=group)
            if _detector == 'WFC': return acsData.WFCInputImage(input,group=group)
            if _detector == 'SBC': return acsData.SBCInputImage(input,group=group)
        if _instrument == 'NICMOS':
            from . import nicmosData
            if _detector == 1: return nicmosData.NIC1InputImage(input)
            if _detector == 2: return nicmosData.NIC2InputImage(input)
            if _detector == 3: return nicmosData.NIC3InputImage(input)


        if _instrument == 'WFPC2':
            from . import wfpc2Data
            return wfpc2Data.WFPC2InputImage(input,group=group)
        """
            if _detector == 1: return wfpc2Data.PCInputImage(input)
            if _detector == 2: return wfpc2Data.WF2InputImage(input)
            if _detector == 3: return wfpc2Data.WF3InputImage(input)
            if _detector == 4: return wfpc2Data.WF4InputImage(input)
        """
        if _instrument == 'STIS':
            from . import stisData
            if _detector == 'CCD': return stisData.CCDInputImage(input,group=group)
            if _detector == 'FUV-MAMA': return stisData.FUVInputImage(input,group=group)
            if _detector == 'NUV-MAMA': return stisData.NUVInputImage(input,group=group)
        if _instrument == 'WFC3':
            from . import wfc3Data
            if _detector == 'UVIS': return wfc3Data.WFC3UVISInputImage(input,group=group)
            if _detector == 'IR': return wfc3Data.WFC3IRInputImage(input,group=group)

    except ImportError:
        msg = 'No module implemented for '+str(_instrument)+'!'
        raise ValueError(msg)
    # If a supported instrument is not detected, print the following error message
    # and raise an exception.
    msg = 'Instrument: ' + str(_instrument) + '/' + str(_detector) + ' not yet supported!'
    raise ValueError(msg)

#### Remaining functions support process_input()
def processFilenames(input=None,output=None,infilesOnly=False):
    """Process the input string which contains the input file information and
       return a filelist,output
    """
    ivmlist = None
    oldasndict = None

    if input is None:
        print("No input files provided to processInput")
        raise ValueError

    if not isinstance(input, list) and ('_asn' in input or '_asc' in input):
        # Input is an association table
        # Get the input files, and run makewcs on them
        oldasndict = asnutil.readASNTable(input, prodonly=infilesOnly)

        if not infilesOnly:
            if output in ["",None,"None"]:
                output = oldasndict['output'].lower() # insure output name is lower case

        asnhdr = fits.getheader(input, memmap=False)
        # Only perform duplication check if not already completed...
        dupcheck = asnhdr.get('DUPCHECK',default="PERFORM") == "PERFORM"

        #filelist = [fileutil.buildRootname(fname) for fname in oldasndict['order']]
        filelist = buildASNList(oldasndict['order'],input,check_for_duplicates=dupcheck)

    elif (not isinstance(input, list)) and \
       (input[0] == '@') :
        # input is an @ file
        f = open(input[1:])
        # Read the first line in order to determine whether
        # IVM files have been specified in a second column...
        line = f.readline()
        f.close()
        # Parse the @-file with irafglob to extract the input filename
        filelist = irafglob.irafglob(input, atfile=util.atfile_sci)
        # If there is a second column...
        if len(line.split()) == 2:
            # ...parse out the names of the IVM files as well
            ivmlist = irafglob.irafglob(input, atfile=util.atfile_ivm)
        if output in ['',None,"None"]:
            if len(filelist) == 1:
                output = fileutil.buildNewRootname(filelist[0])
            else:
                output = 'final'
    else:
        #input is a string or a python list
        try:
            filelist, output = parseinput.parseinput(input, outputname=output)
            if output in ['',None,"None"]:
                if len(filelist) == 1:
                    output = fileutil.buildNewRootname(filelist[0])
                else:
                    output = 'final'
            if not isinstance(input, list):
                filelist.sort()
        except IOError: raise

    # sort the list of input files
    # this ensures the list of input files has the same order on all platforms
    # it can have ifferent order because listdir() uses inode order, not unix type order
    #filelist.sort()



    return filelist, output, ivmlist, oldasndict


def process_input(input, output=None, ivmlist=None, updatewcs=True,
                  prodonly=False,  wcskey=None, **workinplace):
    """
    Create the full input list of filenames after verifying and converting
    files as needed.
    """

    newfilelist, ivmlist, output, oldasndict, origflist = buildFileListOrig(
            input, output=output, ivmlist=ivmlist, wcskey=wcskey,
            updatewcs=updatewcs, **workinplace)

    if not newfilelist:
        buildEmptyDRZ(input, output)
        return None, None, output

    # run all WCS updating -- Now done in buildFileList
    #pydr_input = _process_input_wcs(newfilelist, wcskey, updatewcs)
    pydr_input = newfilelist

    # AsnTable will handle the case when output==None
    if not oldasndict:# and output is not None:
        oldasndict = asnutil.ASNTable(pydr_input, output=output)
        oldasndict.create()

    asndict = update_member_names(oldasndict, pydr_input)
    asndict['original_file_names'] = origflist

    # Build output filename
    drz_extn = '_drz.fits'
    for img in newfilelist:
        # special case logic to automatically recognize when _flc.fits files
        # are provided as input and produce a _drc.fits file instead
        if '_flc.fits' in img:
            drz_extn = '_drc.fits'
            break

    if output in [None,'']:
        output = fileutil.buildNewRootname(asndict['output'],
                                           extn=drz_extn)
    else:
        if '.fits' in output.lower():
            pass
        elif drz_extn[:4] not in output.lower():
            output = fileutil.buildNewRootname(output, extn=drz_extn)


    log.info('Setting up output name: %s' % output)

    return asndict, ivmlist, output


def _process_input_wcs(infiles, wcskey, updatewcs):
    """
    This is a subset of process_input(), for internal use only.  This is the
    portion of input handling which sets/updates WCS data, and is a performance
    hit - a target for parallelization. Returns the expanded list of filenames.
    """

    # Run parseinput though it's likely already been done in processFilenames
    outfiles = parseinput.parseinput(infiles)[0]

    # Disable parallel processing here for now until hardware I/O gets "wider".
    # Since this part is IO bound, parallelizing doesn't help more than a little
    # in most cases, and may actually slow this down on some desktop nodes.
#   cfgval_num_cores = None # get this from paramDict
#   pool_size = util.get_pool_size(cfgval_num_cores, len(outfiles))
    pool_size = 1

    # do the WCS updating
    if wcskey in ['', ' ', 'INDEF', None]:
        if updatewcs:
            log.info('Updating input WCS using "updatewcs"')
    else:
        log.info('Resetting input WCS to be based on WCS key = %s' % wcskey)

    if pool_size > 1:
        log.info('Executing %d parallel workers' % pool_size)
        subprocs = []
        for fname in outfiles:
            p = multiprocessing.Process(target=_process_input_wcs_single,
                name='processInput._process_input_wcs()', # for err msgs
                args=(fname, wcskey, updatewcs) )
            subprocs.append(p)
        mputil.launch_and_wait(subprocs, pool_size) # blocks till all done
    else:
        log.info('Executing serially')
        for fname in outfiles:
            _process_input_wcs_single(fname, wcskey, updatewcs)

    return outfiles


def _process_input_wcs_single(fname, wcskey, updatewcs):
    """
    See docs for _process_input_wcs.
    This is separated to be spawned in parallel.
    """
    if wcskey in ['', ' ', 'INDEF', None]:
        if updatewcs:
            uw.updatewcs(fname, checkfiles=False)
    else:
        numext = fileutil.countExtn(fname)
        extlist = []
        for extn in range(1, numext + 1):
            extlist.append(('SCI', extn))
        if wcskey in string.ascii_uppercase:
            wkey = wcskey
            wname = ' '
        else:
            wname = wcskey
            wkey = ' '
        altwcs.restoreWCS(fname, extlist, wcskey=wkey, wcsname=wname)
    # make an asn table at the end
    # Make sure there is a WCSCORR table for each input image
    if wcskey not in ['', ' ', 'INDEF', None] or updatewcs:
        wcscorr.init_wcscorr(fname)


def buildFileList(input, output=None, ivmlist=None,
                wcskey=None, updatewcs=True, **workinplace):
    """
    Builds a file list which has undergone various instrument-specific
    checks for input to MultiDrizzle, including splitting STIS associations.
    """
    newfilelist, ivmlist, output, oldasndict, filelist = \
        buildFileListOrig(input=input, output=output, ivmlist=ivmlist,
                    wcskey=wcskey, updatewcs=updatewcs, **workinplace)
    return newfilelist, ivmlist, output, oldasndict


def buildFileListOrig(input, output=None, ivmlist=None,
                wcskey=None, updatewcs=True, **workinplace):
    """
    Builds a file list which has undergone various instrument-specific
    checks for input to MultiDrizzle, including splitting STIS associations.
    Compared to buildFileList, this version returns the list of the
    original file names as specified by the user (e.g., before GEIS->MEF, or
    WAIVER FITS->MEF conversion).
    """
    # NOTE: original file name is required in order to correctly associate
    # user catalog files (e.g., user masks to be used with 'skymatch') with
    # corresponding imageObjects.

    filelist, output, ivmlist, oldasndict = processFilenames(input,output)

    # verify that all input images specified can be updated as needed
    filelist = util.verifyFilePermissions(filelist)
    if filelist is None or len(filelist) == 0:
        return None, None, None, None, None

    manageInputCopies(filelist,**workinplace)

    # to keep track of the original file names we do the following trick:
    # pack filelist with the ivmlist using zip and later unpack the zipped list.
    #
    # NOTE: this required a small modification of the checkStisFiles function
    # in stsci.tools.check_files to be able to handle ivmlists that are tuples.
    if ivmlist is None:
        ivmlist = len(filelist)*[None]
    else:
        assert(len(filelist) == len(ivmlist)) #TODO: remove after debugging
    ivmlist = list(zip(ivmlist,filelist))

    # Check format of FITS files - convert Waiver/GEIS to MEF if necessary
    filelist, ivmlist = check_files.checkFITSFormat(filelist, ivmlist)

    # check for non-polynomial distortion correction
    if not updatewcs:
        # with updatewcs turned on, any problems will get resolved
        # so we do not need to be concerned about the state of the DGEOFILEs
        filelist = checkDGEOFile(filelist)

    # run all WCS updating
    updated_input = _process_input_wcs(filelist, wcskey, updatewcs)

    newfilelist, ivmlist = check_files.checkFiles(updated_input, ivmlist)

    if updatewcs:
        uw.updatewcs(','.join(set(newfilelist) - set(filelist)))

    if len(ivmlist) > 0:
        ivmlist, filelist = list(zip(*ivmlist))
    else:
        filelist = [] # insure that both filelist and ivmlist are defined as empty lists

    return newfilelist, ivmlist, output, oldasndict, filelist


def buildASNList(rootnames, asnname, check_for_duplicates=True):
    """
    Return the list of filenames for a given set of rootnames
    """

    # Recognize when multiple valid inputs with the same rootname are present
    # this would happen when both CTE-corrected (_flc) and non-CTE-corrected (_flt)
    # products are in the same directory as an ASN table
    filelist, duplicates = checkForDuplicateInputs(rootnames)

    if check_for_duplicates and duplicates:
        # Build new ASN tables for each set of input files
        origasn = changeSuffixinASN(asnname, 'flt')
        dupasn = changeSuffixinASN(asnname, 'flc')

        errstr = 'ERROR:\nMultiple valid input files found:\n'
        for fname, dname in zip(filelist, duplicates):
            errstr += '    %s    %s\n' % (fname, dname)
        errstr += ('\nNew association files have been generated for each '
                   'version of these files.\n    %s\n    %s\n\nPlease '
                   're-start astrodrizzle using of these new ASN files or '
                   'use widlcards for the input to only select one type of '
                   'input file.' % (dupasn, origasn))

        print(textutil.textbox(errstr), file=sys.stderr)

        # generate new ASN files for each case,
        # report this case of duplicate inputs to the user then quit
        raise ValueError

    return filelist


def changeSuffixinASN(asnfile, suffix):
    """
    Create a copy of the original asn file and change the name of all members
    to include the suffix.
    """
    # Start by creating a new name for the ASN table
    _new_asn = asnfile.replace('_asn.fits','_'+suffix+'_asn.fits')
    if os.path.exists(_new_asn):
        os.remove(_new_asn)
    # copy original ASN table to new table
    shutil.copy(asnfile,_new_asn)

    # Open up the new copy and convert all MEMNAME's to include suffix
    fasn = fits.open(_new_asn, mode='update', memmap=False)
    fasn[0].header['DUPCHECK'] = "COMPLETE"
    newdata = fasn[1].data.tolist()
    for i in range(len(newdata)):
        val = newdata[i][0].decode(encoding='UTF-8').strip()
        if 'prod' not in newdata[i][1].decode(encoding='UTF-8').lower():
            val += '_'+suffix
        newdata[i] = (val,newdata[i][1].strip(),newdata[i][2])

    # Redefine dtype to support longer strings for MEMNAME
    new_dtype = []
    d = fasn[1].data.dtype
    msize = d.descr[0][1][1:]
    new_size = int(msize[1:])+8
    mtype = msize[0]
    new_dtype.append((d.descr[0][0],d.descr[0][1].replace(msize,'{}{}'.format(mtype,new_size))))
    new_dtype.append(d.descr[1])
    new_dtype.append(d.descr[2])

    # Assign newly created, reformatted array to extension
    newasn = np.array(newdata,dtype=new_dtype)
    fasn[1].data = newasn
    fasn.close()

    return _new_asn


def checkForDuplicateInputs(rootnames):
    """
    Check input files specified in ASN table for duplicate versions with
    multiple valid suffixes (_flt and _flc, for example).
    """

    flist = []
    duplist = []

    for fname in rootnames:
        # Look for any recognized CTE-corrected products
        f1 = fileutil.buildRootname(fname,ext=['_flc.fits'])
        f2 = fileutil.buildRootname(fname)
        flist.append(f2)
        if os.path.exists(f1) and f1 != f2:
            # More than 1 valid input found for this rootname
            duplist.append(f1)

    return flist,duplist


def runmakewcs(input):
    """
    Runs make wcs and recomputes the WCS keywords

    Parameters
    ----------
    input : str or list of str
        a list of files

    Returns
    -------
    output : list of str
        returns a list of names of the modified files
        (For GEIS files returns the translated names.)
    """
    newNames = uw.updatewcs(input, checkfiles=False)
    #newNames = makewcs.run(input)
    return newNames


def resetDQBits(imageObjectList, cr_bits_value=4096):
    """Reset the CR bit in each input image's DQ array"""

    if cr_bits_value > 0:
        for img in imageObjectList:
            for chip in range(1,img._numchips+1,1):
                sci_chip = img._image[img.scienceExt,chip]
                resetbits.reset_dq_bits(sci_chip.dqfile, cr_bits_value,
                                        extver=chip, extname=sci_chip.dq_extn)


def update_member_names(oldasndict, pydr_input):
    """
    Update names in a member dictionary.

    Given an association dictionary with rootnames and a list of full
    file names, it will update the names in the member dictionary to
    contain '_*' extension. For example a rootname of 'u9600201m' will
    be replaced by 'u9600201m_c0h' making sure that a MEf file is passed
    as an input and not the corresponding GEIS file.
    """

    omembers = oldasndict['members'].copy()
    nmembers = {}
    translated_names = [f.split('.fits')[0] for f in pydr_input]

    newkeys = [fileutil.buildNewRootname(file) for file in pydr_input]
    keys_map = list(zip(newkeys, pydr_input))

    for okey, oval in list(omembers.items()):
        if okey in newkeys:
            nkey = pydr_input[newkeys.index(okey)]
            nmembers[nkey.split('.fits')[0]] = oval

    oldasndict.pop('members')
    # replace should be always True to cover the case when flt files were removed
    # and the case when names were translated

    oldasndict.update(members=nmembers, replace=True)
    oldasndict['order'] = translated_names
    return oldasndict


def manageInputCopies(filelist, **workinplace):
    """
    Creates copies of all input images in a sub-directory.

    The copies are made prior to any processing being done to the images at all,
    including updating the WCS keywords. If there are already copies present,
    they will NOT be overwritten, but instead will be used to over-write the
    current working copies.
    """

    # Find out what directory is being used for processing
    workingdir = os.getcwd()
    # Only create sub-directory for copies of inputs, if copies are requested
    # Create name of sub-directory for copies
    origdir = os.path.join(workingdir,'OrIg_files')
    if workinplace['overwrite'] or workinplace['preserve']:
        # if sub-directory does not exist yet, create it
        if not os.path.exists(origdir):
            os.mkdir(origdir)

    printMsg = True
    # check to see if copies already exist for each file
    for fname in filelist:
        copymade = False # If a copy is made, no need to restore
        copyname = os.path.join(origdir,fname)
        short_copyname = os.path.join('OrIg_files',fname)
        if workinplace['overwrite']:
            print('Forcibly archiving original of: ',fname, 'as ',short_copyname)
            # make a copy of the file in the sub-directory
            if os.path.exists(copyname): os.chmod(copyname, 438) # octal 666
            shutil.copy(fname,copyname)
            os.chmod(copyname,292) # octal 444 makes files read-only
            if printMsg:
                print('\nTurning OFF "preserve" and "restore" actions...\n')
                printMsg = False # We only need to print this one time...
            copymade = True

        if (workinplace['preserve'] and not os.path.exists(copyname)) \
                and not workinplace['overwrite']:
            # Preserving a copy of the input, but only if not already archived
            print('Preserving original of: ',fname, 'as ',short_copyname)
            # make a copy of the file in the sub-directory
            shutil.copy(fname,copyname)
            os.chmod(copyname,292) # octal 444 makes files read-only
            copymade = True

        if 'restore' in workinplace and not copymade:
            if (os.path.exists(copyname) and workinplace['restore']) and not workinplace['overwrite']:
                print('Restoring original input for ',fname,' from ',short_copyname)
                # replace current files with original version
                os.chmod(fname, 438) # octal 666
                shutil.copy(copyname, fname)
                os.chmod(fname, 438) # octal 666


def buildEmptyDRZ(input, output):
    """
    Create an empty DRZ file.

    This module creates an empty DRZ file in a valid FITS format so that the HST
    pipeline can handle the Multidrizzle zero expossure time exception
    where all data has been excluded from processing.

    Parameters
    ----------
    input : str
        filename of the initial input to process_input
    output : str
        filename of the default empty _drz.fits file to be generated

    """

    # Identify the first input image
    inputfile = parseinput.parseinput(input)[0]
    if not inputfile:
        print('\n******* ERROR *******', file=sys.stderr)
        print(
              'No input file found!  Check specification of parameter '
              '"input". ', file=sys.stderr)
        print('Quitting...',  file=sys.stderr)
        print('******* ***** *******\n',  file=sys.stderr)
        return # raise IOError, "No input file found!"

    # Set up output file here...
    if output is None:
        if len(input) == 1:
            oname = fileutil.buildNewRootname(input[0])
        else:
            oname = 'final'
        output = fileutil.buildNewRootname(oname, extn='_drz.fits')
    else:
        if 'drz' not in output:
            output = fileutil.buildNewRootname(output, extn='_drz.fits')

    log.info('Setting up output name: %s' % output)

    # Open the first image (of the excludedFileList?) to use as a template to build
    # the DRZ file.
    try :
        log.info('Building empty DRZ file from %s' % inputfile[0])
        img = fits.open(inputfile[0], memmap=False)
    except:
        raise IOError('Unable to open file %s \n' % inputfile)

    # Create the fitsobject
    fitsobj = fits.HDUList()
    # Copy the primary header
    hdu = img[0].copy()
    fitsobj.append(hdu)

    # Modify the 'NEXTEND' keyword of the primary header to 3 for the
    #'sci, wht, and ctx' extensions of the newly created file.
    fitsobj[0].header['NEXTEND'] = 3

    # Create the 'SCI' extension
    hdu = fits.ImageHDU(header=img['sci', 1].header.copy())
    hdu.header['EXTNAME'] = 'SCI'
    fitsobj.append(hdu)

    # Create the 'WHT' extension
    hdu = fits.ImageHDU(header=img['sci', 1].header.copy())
    hdu.header['EXTNAME'] = 'WHT'
    fitsobj.append(hdu)

    # Create the 'CTX' extension
    hdu = fits.ImageHDU(header=img['sci', 1].header.copy())
    hdu.header['EXTNAME'] = 'CTX'
    fitsobj.append(hdu)

    # Add HISTORY comments explaining the creation of this file.
    fitsobj[0].header.add_history("** AstroDrizzle has created this empty "
                                  "DRZ product because**")
    fitsobj[0].header.add_history("** all input images were excluded from "
                                  "processing.**")


    # Change the filename in the primary header to reflect the name of the output
    # filename.
    fitsobj[0].header['FILENAME'] = str(output)  # +"_drz.fits"

    # Change the ROOTNAME keyword to the ROOTNAME of the output PRODUCT
    fitsobj[0].header['ROOTNAME'] = str(output.split('_drz.fits')[0])
    # Modify the ASN_MTYP keyword to contain "PROD-DTH" so it can be properly
    # ingested into the archive catalog.

    # stis has this keyword in the [1] header, so I am directing the code
    #t o first look in the primary, then the 1
    try:
        fitsobj[0].header['ASN_MTYP'] = 'PROD-DTH'
    except:
        fitsobj[1].header['ASN_MTYP'] = 'PROD-DTH'

    # If the file is already on disk delete it and replace it with the
    # new file
    dirfiles = os.listdir(os.curdir)
    if dirfiles.count(output) > 0:
        os.remove(output)
        log.info("       Replacing %s..." % output)

    # Write out the empty DRZ file
    fitsobj.writeto(output)

    print(textutil.textbox(
        'ERROR:\nAstroDrizzle has created an empty DRZ product because all '
        'input images were excluded from processing or a user requested the '
        'program to stop.') + '\n', file=sys.stderr)

    return


def checkDGEOFile(filenames):
    """
    Verify that input file has been updated with NPOLFILE

    This function checks for the presence of 'NPOLFILE' kw in the primary header
    when 'DGEOFILE' kw is present and valid (i.e. 'DGEOFILE' is not blank or 'N/A').
    It handles the case of science files downloaded from the archive before the new
    software was installed there.
    If 'DGEOFILE' is present and 'NPOLFILE' is missing, print a message and let the user
    choose whether to (q)uit and update the headers or (c)ontinue and run astrodrizzle
    without the non-polynomial correction.
    'NPOLFILE' will be populated in the pipeline before astrodrizzle is run.

    In the case of WFPC2 the old style dgeo files are used to create detector to image
    correction at runtime.

    Parameters
    ----------
    filenames : list of str
        file names of all images to be checked

    """

    msg = """
            A 'DGEOFILE' keyword is present in the primary header but 'NPOLFILE' keyword was not found.
            This version of the software uses a new format for the residual distortion DGEO files.
            Please consult the instrument web pages for which reference files to download.
            A small (new style) dgeofile is needed ('_npl.fits' extension) and possibly a
            detector to image correction file ('_d2i.fits' extension).
            The names of these files must be added to the primary header either using the task XXXX
            or manually, for example:

            hedit {0:s}[0] npolfile fname_npl.fits add+
            hedit {0:s}[0] d2imfile fname_d2i.fits add+

            where fname_npl.fits is the name of the new style dgeo file and fname_d2i.fits is
            the name of the detector to image correction. After adding these keywords to the
            primary header, updatewcs must be run to update the science files:

            from stwcs import updatewcs
            updatewcs.updatewcs("{0:s}")

            Alternatively you may choose to run astrodrizzle without DGEO and detector to image correction.

            To stop astrodrizzle and update the dgeo files, type 'q'.
            To continue running astrodrizzle without the non-polynomial distortion correction, type 'c':
            """

    short_msg = """
            To stop astrodrizzle and update the dgeo files, type 'q'.
            To continue running astrodrizzle without the non-polynomial distortion correction, type 'c':
    """

    for inputfile in filenames:
        try:
            dgeofile = fits.getval(inputfile, 'DGEOFILE', memmap=False)
        except KeyError:
            continue
        if dgeofile not in ["N/A", "n/a", ""]:
            message = msg.format(inputfile)
            try:
                npolfile = fits.getval(inputfile, 'NPOLFILE', memmap=False)
            except KeyError:
                ustop = userStop(message)
                while ustop is None:
                    ustop = userStop(short_msg)
                if ustop:
                    return None

    return filenames

def userStop(message):
    if sys.version_info[0] >= 3:
        user_input = input(message)
    else:
        user_input = raw_input(message)

    if user_input == 'q':
        return True
    elif user_input == 'c':
        return False
    else:
        return None

def _setDefaults(input_dict={}):
    """ Define full set of default values for unit-testing this module.[OBSOLETE]"""
    paramDict = {
        'input':'*flt.fits',
        'output':None,
        'mdriztab':None,
        'refimage':None,
        'runfile':None,
        'workinplace':False,
        'updatewcs':True,
        'proc_unit':'native',
        'coeffs':True,
        'context':False,
        'clean':True,
        'group':None,
        'ra':None,
        'dec':None,
        'build':True,
        'gain':None,
        'gnkeyword':None,
        'readnoise':None,
        'rnkeyword':None,
        'exptime':None,
        'expkeyword':None,
        'crbitval':4096,
        'static':True,
        'static_sig':4.0,
        'skysub':True,
        'skymethod':"globalmin+match",
        'skystat':"median",
        'skywidth':0.1,
        'skylower':None,
        'skyupper':None,
        'skyclip':5,
        'skylsigma':4.0,
        'skyusigma':4.0,
        "skymask_cat":"",
        "use_static":True,
        "sky_bits":0,
        "skyuser":"",
        "skyfile":"",
        'driz_separate':True,
        'driz_sep_outnx':None,
        'driz_sep_outny':None,
        'driz_sep_crpix1':None,
        'driz_sep_crpix2':None,
        'driz_sep_kernel':'turbo',
        'driz_sep_scale':None,
        'driz_sep_pixfrac':1.0,
        'driz_sep_rot':None,
        'driz_sep_fillval':None,
        'driz_sep_bits':0,
        'median':True,
        'median_newmasks':True,
        'combine_type':"minmed",
        'combine_nsigma':"4 3",
        'combine_nlow':0,
        'combine_nhigh':1,
        'combine_lthresh':None,
        'combine_hthresh':None,
        'combine_grow':1,
        'blot':True,
        'blot_interp':'poly5',
        'blot_sinscl':1.0,
        'driz_cr':True,
        'driz_cr_corr':False,
        'driz_cr_snr':"3.5 3.0",
        'driz_cr_scale':"1.2 0.7",
        'driz_cr_cteg':0,
        'driz_cr_grow':1,
        'driz_combine':True,
        'final_wht_type':"EXP",
        'final_outnx':None,
        'final_outny':None,
        'final_crpix1':None,
        'final_crpix2':None,
        'final_kernel':'square',
        'final_scale':None,
        'final_pixfrac':1.0,
        'final_rot':None,
        'final_fillval':None,
        'final_bits':0}

    paramDict.update(input_dict)

    print('\nUser Input Parameters for Init Step:')
    util.printParams(paramDict)

    return paramDict
