"""Code that evaluates the quality of the SVM products generated by the drizzlepac package.

The JSON files generated here can be converted directly into a Pandas DataFrame
using the syntax:

>>> import json
>>> import pandas as pd
>>> with open("<rootname>_astrometry_resids.json") as jfile:
>>>     resids = json.load(jfile)
>>> pdtab = pd.DataFrame(resids)

These DataFrames can then be concatenated using:

>>> allpd = pdtab.concat([pdtab2, pdtab3])

where 'pdtab2' and 'pdtab3' are DataFrames generated from other datasets.  For
more information on how to merge DataFrames, see

https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html

Visualization of these Pandas DataFrames with Bokeh can follow the example
from:

https://programminghistorian.org/en/lessons/visualizing-with-bokeh

PUBLIC INTERFACE FOR THIS MODULE
 - build_svm_plots(HDF5_FILE, output_basename='', display_plot=False):

"""

# Standard library imports
import argparse
import collections
from datetime import datetime
import glob
import json
import logging
import os
import pdb
import pickle
import re
import sys
import time

# Related third party imports
from astropy.coordinates import SkyCoord
from astropy.io import ascii, fits
from astropy.stats import sigma_clipped_stats
from astropy.table import Table
from itertools import chain
import numpy as np
import pandas as pd
from scipy.spatial import KDTree

from bokeh.layouts import row, column, gridplot
from bokeh.plotting import figure, output_file, save, show
from bokeh.models import ColumnDataSource, Label, CDSView, Div
from bokeh.models.tools import HoverTool

# Local application imports
from drizzlepac import util, wcs_functions
import drizzlepac.devutils.comparison_tools.compare_sourcelists as csl
from drizzlepac.haputils.graph_utils import HAPFigure, build_tooltips
from drizzlepac.haputils.pandas_utils import PandasDFReader, get_pandas_data
from drizzlepac.haputils.pipeline_graphics import build_vector_plot
from drizzlepac.devutils.comparison_tools.read_hla import read_hla_catalog
from stsci.tools import logutil
from stwcs import wcsutil
from stwcs.wcsutil import HSTWCS


__taskname__ = 'svm_quality_graphics'

MSG_DATEFMT = '%Y%j%H%M%S'
SPLUNK_MSG_FORMAT = '%(asctime)s %(levelname)s src=%(name)s- %(message)s'
log = logutil.create_logger(__name__, level=logutil.logging.NOTSET, stream=sys.stdout,
                            format=SPLUNK_MSG_FORMAT, datefmt=MSG_DATEFMT)
# ----------------------------------------------------------------------------------------------------------------------
# Module level variables

# ====================================================================================
# GAIA plots: number of GAIA sources, mean distance to neighbors, centroid/offset/std
# of sources in field
# ====================================================================================


def build_svm_plots(data_source, output_basename='', display_plot=False, log_level=logutil.logging.INFO):
    """Create all the plots for the results generated by these comparisons

    Parameters
    ----------
    data_source : str
        Filename for master data file which contains all the results.  This will
        typically be an HSF5 file generated by the JSON harvester.

    output_base_filename : str
        Base name for the HMTL file generated by Bokeh.

    display_plot : bool, optional
        Option to display the plot to the screen
        Default: False

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    -------
    Nothing.
    """
    log.setLevel(log_level)
    if output_basename == '':
        output_basename = "svm_qa"
    else:
        output_basename = "{}_svm_qa".format(output_basename)

    # Column names as defined in the harvester dataframe mapped to simple names for ease of use
    gaia_col_names = {'distribution_characterization_statistics.Number_of_GAIA_sources': 'num_GAIA',
                      'distribution_characterization_statistics.X_centroid': 'x_centroid',
                      'distribution_characterization_statistics.X_offset': 'x_offset',
                      'distribution_characterization_statistics.X_standard_deviation': 'x_std',
                      'distribution_characterization_statistics.Y_centroid': 'y_centroid',
                      'distribution_characterization_statistics.Y_offset': 'y_offset',
                      'distribution_characterization_statistics.Y_standard_deviation': 'y_std',
                      'distribution_characterization_statistics.maximum_neighbor_distance': 'max_neighbor_dist',
                      'distribution_characterization_statistics.mean_neighbor_distance': 'mean_neighbor_dist',
                      'distribution_characterization_statistics.minimum_neighbor_distance': 'min_neighbor_dist',
                      'distribution_characterization_statistics.standard_deviation_of_neighbor_distances': 'std_neighbor_dist'}

    # Get the requested columns from the dataframe in addition columns
    # added by the pandas_utils
    gaia_cols_DF = get_pandas_data(data_source, gaia_col_names.keys())

    # Rename the columns to abbreviated text for ease of management
    for old_col_name, new_col_name in gaia_col_names.items():
        gaia_cols_DF.rename(columns={old_col_name: new_col_name}, inplace=True)

    gaia_plots_name = build_gaia_plots(gaia_cols_DF, list(gaia_col_names.values()), display_plot,
                                       output_basename=output_basename, log_level=log_level)

    #     -      -     -      -     -      -     -      -     -      -     -      -     -      -     -      -
    # Generate plots for interfilter cross-match comparisons
    # Column names as defined in the harvester dataframe mapped to simple names for ease of use
    intfilt_xm_col_names = {'Interfilter_cross-match_details.reference_image_name': 'ref_imgname',
                            'Interfilter_cross-match_details.comparison_image_name': 'comp_imgname',
                            'Interfilter_cross-match_details.reference_catalog_length': 'ref_catalog_length',
                            'Interfilter_cross-match_details.comparison_catalog_length': 'comp_catalog_length',
                            'Interfilter_cross-match_details.number_of_cross-matches': 'number_of_cross-matches',
                            'Interfilter_cross-match_details.percent_of_all_identified_reference_sources_crossmatched': 'ref_crossmatch_percentage',
                            'Interfilter_cross-match_details.percent_of_all_identified_comparison_sources_crossmatched': 'comp_crossmatch_percentage',
                            'Interfilter_cross-match_details.reference_image_platescale': 'ref_image_platescale',
                            'Interfilter_cross-matched_reference_catalog.xcentroid_ref': 'ref_catalog.xcentroid_ref',
                            'Interfilter_cross-matched_reference_catalog.ycentroid_ref': 'ref_catalog.ycentroid_ref',
                            'Interfilter_cross-matched_comparison_catalog.xcentroid_ref': 'comp_catalog.xcentroid_ref',
                            'Interfilter_cross-matched_comparison_catalog.ycentroid_ref': 'comp_catalog.ycentroid_ref',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_min': 'x_non-clipped_min',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_max': 'x_non-clipped_max',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_mean': 'x_non-clipped_mean',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_median': 'x_non-clipped_median',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_standard_deviation': 'x_non-clipped_rms',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.3x3_sigma-clipped_mean': 'x_3x3_sigma-clipped_mean',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.3x3_sigma-clipped_median': 'x_3x3_sigma-clipped_median',
                            'Interfilter_cross-matched_xcentroid_ref_comparison_-_reference_separation_statistics.3x3_sigma-clipped_standard_deviation': 'x_3x3_sigma-clipped_rms',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_min': 'y_non-clipped_min',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_max': 'y_non-clipped_max',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_mean': 'y_non-clipped_mean',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_median': 'y_non-clipped_median',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.Non-clipped_standard_deviation': 'y_non-clipped_rms',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.3x3_sigma-clipped_mean': 'y_3x3_sigma-clipped_mean',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.3x3_sigma-clipped_median': 'y_3x3_sigma-clipped_median',
                            'Interfilter_cross-matched_ycentroid_ref_comparison_-_reference_separation_statistics.3x3_sigma-clipped_standard_deviation': 'y_3x3_sigma-clipped_rms',
                            'Interfilter_cross-matched_comparison_-_reference_separations.delta_xcentroid_ref': 'comp-ref_x_seperation',
                            'Interfilter_cross-matched_comparison_-_reference_separations.delta_ycentroid_ref': 'comp-ref_y_seperation'}

    # Get the requested columns from the dataframe in addition columns
    # added by the pandas_utils
    if_xm_DF = get_pandas_data(data_source, intfilt_xm_col_names.keys())

    # Rename the columns to abbreviated text for ease of management
    for old_col_name, new_col_name in intfilt_xm_col_names.items():
        if_xm_DF.rename(columns={old_col_name: new_col_name}, inplace=True)

    # remove rows that aren't relevant
    if_xm_DF = if_xm_DF[np.isnan(if_xm_DF.ref_image_platescale) == False]

    # Create interfilter plots, or bail out if no rows remain
    if if_xm_DF.shape[0] > 0:
        if_xm_plots_name = build_interfilter_crossmatch_plots(if_xm_DF,
                                                              display_plot,
                                                              output_basename=output_basename,
                                                              log_level=log_level)
    else:
        log.warning("No interfilter residual data to plot!")
        return

    #     -      -     -      -     -      -     -      -     -      -     -      -     -      -     -      -
    # Generate plots for point-segment catalog cross-match comparisons
    xmatch_col_names = ['Cross-match_details.number_of_cross-matches',
                        'Cross-match_details.point_catalog_filename',
                        'Cross-match_details.point_catalog_length',
                        'Cross-match_details.point_frame',
                        'Cross-match_details.segment_catalog_filename',
                        'Cross-match_details.segment_catalog_length',
                        'Cross-match_details.segment_frame',
                        'Cross-matched_point_catalog.Declination',
                        'Cross-matched_point_catalog.Right ascension',
                        'Cross-matched_segment_catalog.Declination',
                        'Cross-matched_segment_catalog.Right ascension',
                        'Segment_-_point_on-sky_separation_statistics.3x3_sigma-clipped_mean',
                        'Segment_-_point_on-sky_separation_statistics.3x3_sigma-clipped_median',
                        'Segment_-_point_on-sky_separation_statistics.3x3_sigma-clipped_standard_deviation',
                        'Segment_-_point_on-sky_separation_statistics.Non-clipped_max',
                        'Segment_-_point_on-sky_separation_statistics.Non-clipped_mean',
                        'Segment_-_point_on-sky_separation_statistics.Non-clipped_median',
                        'Segment_-_point_on-sky_separation_statistics.Non-clipped_min',
                        'Segment_-_point_on-sky_separation_statistics.Non-clipped_standard_deviation']

    xmatch_cols = get_pandas_data(data_source, xmatch_col_names)

    xmatch_plots_name = build_crossmatch_plots(xmatch_cols, xmatch_col_names,
                                               output_basename=output_basename,
                                               display_plot=display_plot,
                                               log_level=log_level)

    # Generate the WCS comparison graphics - compares the Primary WCS to the alternate WCS
    wcs_graphics_driver(data_source, output_basename, display_plot, log_level=log_level)

    #
    # Catalog comparisons
    #

    # Comparison of number of sources detected in the Point and Segment catalogs
    nsources_graphics_driver(data_source, output_basename, display_plot, log_level=log_level)

    # Comparison of photometry measurements for Aperture 1 and Apeture 2 between
    # Point and Segment catalogs
    photometry_graphics_driver(data_source, output_basename, display_plot, log_level=log_level)


# -----------------------------------------------------------------------------
# Functions for generating each data plot
#
def build_gaia_plots(gaiaDF, data_cols, display_plot, output_basename='svm_qa', log_level=logging.INFO):
    """
    Generate the plots for evaluating the distribution of GAIA catalog sources
    in the field-of-view of each product.

    Parameters
    ----------
    gaiaDF : Pandas dataframe
        This dataframe contains all the columns relevant to the plots.

    data_cols : list
        A subset of the column names in gaiaDF

    output_basename : str
        String to use as the start of the filename for the output plot pages.

    display_plot : bool
        Option to display the plot in addition to writing out the file.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.
    Returns
    -------
    output : str
        Name of HTML file where the plot was saved.

    """
    log.setLevel(log_level)
    # Setup the source of the data to be plotted so the axis variables can be
    # referenced by column name in the Pandas dataframe
    gaiaCDS = ColumnDataSource(gaiaDF)
    num_of_datasets = len(gaiaCDS.data['index'])
    log.info('Number of datasets: {}'.format(num_of_datasets))

    output_basename = "{}_gaia_comparison".format(output_basename)

    if not output_basename.endswith('.html'):
        output = output_basename + '.html'
    else:
        output = output_basename
    # Set the output file immediately as advised by Bokeh.
    output_file(output)

    # Generate the graphic-specific tooltips - be mindful of
    # the default tooltips defined in graph_utils.py
    gaia_tooltips_list = ['DATE', 'RA', 'DEC', 'GYRO', 'EXPTIME',
                          'ALIGNED_TO']

    gaia_hover_columns = ['header.DATE-OBS',
                          'header.RA_TARG',
                          'header.DEC_TARG',
                          'header.GYROMODE',
                          'header.EXPTIME',
                          'fit_results.aligned_to']

    gaia_tips = build_tooltips(gaia_tooltips_list, gaia_hover_columns,
                               list(range(0, len(gaia_hover_columns))))

    # Define the graphics
    # Create title text at the top of the html file
    html_title_text = Div(text="""
    <h1>Distribution characteristics of GAIA catalog sources identified in the observation footprint</h1>""")

    # Scatter figures
    p0 = HAPFigure(title='Centroid of GAIA Sources in Field',
                   x_label='X Centroid (pixels)',
                   y_label='Y Centroid (pixels)',
                   hover_tips=gaia_tips)
    p0.build_glyph('circle',
                   x='x_centroid',
                   y='y_centroid',
                   sourceCDS=gaiaCDS,
                   glyph_color='colormap',
                   legend_group='inst_det')

    p1 = HAPFigure(title='Offset of Centroid of GAIA Sources in Field',
                   x_label='X Offset (pixels)',
                   y_label='Y Offset (pixels)',
                   hover_tips=gaia_tips)
    p1.build_glyph('circle',
                   x='x_offset',
                   y='y_offset',
                   sourceCDS=gaiaCDS,
                   glyph_color='colormap',
                   legend_group='inst_det')

    p2 = HAPFigure(title='Standard Deviation of GAIA Source Positions in Field',
                   x_label='STD(X) (pixels)',
                   y_label='STD(Y) (pixels)',
                   hover_tips=gaia_tips)
    p2.build_glyph('circle',
                   x='x_std',
                   y='y_std',
                   sourceCDS=gaiaCDS,
                   glyph_color='colormap',
                   legend_group='inst_det')

    # Histogram figures
    """
    p3 = HAPFigure(title='Mean distance between GAIA sources in Field',
                   xlabel='Separation (pixels)',
                   ylabel='Number of products',
                   use_hover_tips=False,
                   background_fill_color='gainsboro',
                   toolbar_location='right',
                   ystart=0,
                   grid_line_color='white')
    mean_dist = gaiaDF.loc[:, 'mean_neighbor_dist'].dropna()
    hist3, edges3 = np.histogram(mean_dist, bins=50)
    p3.build_histogram(top=hist3,
                       bottom=0,
                       left=edges3[:-1],
                       right=edges3[1:],
                       fill_color='navy',
                       fill_transparency=0.5,
                       line_color='white')
    """

    p4 = HAPFigure(title='Number of GAIA sources in Field',
                   xlabel='Number of GAIA sources',
                   ylabel='Number of products',
                   use_hover_tips=False,
                   background_fill_color='gainsboro',
                   toolbar_location='right',
                   ystart=0,
                   grid_line_color='white')
    num_of_GAIA_sources = gaiaDF.loc[:, 'num_GAIA'].dropna()
    hist4, edges4 = np.histogram(num_of_GAIA_sources, bins=50)
    p4.build_histogram(top=hist4,
                       bottom=0,
                       left=edges4[:-1],
                       right=edges4[1:],
                       fill_color='navy',
                       fill_transparency=0.5,
                       line_color='white')

    # Display and save
    if display_plot:
        show(column(html_title_text, p4.fig, p0.fig, p1.fig, p2.fig))
        # show(column(p4.fig, p0.fig, p1.fig, p2.fig, p3.fig))
    # Just save
    else:
        save(column(html_title_text, p4.fig, p0.fig, p1.fig, p2.fig))
        # save(column(p4.fig, p0.fig, p1.fig, p2.fig, p3.fig))
    log.info("Output HTML graphic file {} has been written.\n".format(output))

    # Return the name of the plot layout file just created
    return output


def build_crossmatch_plots(xmatchDF, data_cols,
                           output_basename='svm_qa',
                           display_plot=False,
                           log_level=logging.INFO):
    """Generate the cross-match statistics plots for the comparison between the
    point catalog and the segment catalog.

    Parameters
    ----------
    xmatchDF : Pandas DataFrame
        This object contains all the columns relevant to the cross-match plots.

    data_cols : list
        The list of column names for the columns read in to the `xmatchCDS` object.

    output_basename : str
        String to use as the start of the filename for the output plot pages.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    -------
    output : str
        Name of HTML file where the plot was saved.
        
    """
    log.setLevel(log_level)

    xmatchCDS = ColumnDataSource(xmatchDF)

    output_basename = "{}_point-segment_comparison".format(output_basename)

    if not output_basename.endswith('.html'):
        output = output_basename + '.html'
    else:
        output = output_basename
    # Set the output file immediately as advised by Bokeh.
    output_file(output)
        
    # Convert the data into histograms now...
    p0 = HAPFigure(title='Number of Point-to-Segment Cross-matched sources',
                   xlabel='Number of Cross-matched sources',
                   ylabel='Number of products',
                   use_hover_tips=False,
                   background_fill_color='gainsboro',
                   toolbar_location='right',
                   ystart=0,
                   grid_line_color='white')
    data0 = xmatchCDS.data[data_cols[0]]
    data0 = data0[~np.isnan(data0)]
    hist0, edges0 = np.histogram(data0, bins=50)
    p0.build_histogram(top=hist0,
                       bottom=0,
                       left=edges0[:-1],
                       right=edges0[1:],
                       fill_color='navy',
                       fill_transparency=0.5,
                       line_color='white')

    title1 = 'Mean Separation (Sigma-clipped) of Point-to-Segment Cross-matched sources'
    p1 = HAPFigure(title=title1,
                   xlabel='Mean Separation of Cross-matched sources (arcseconds)',
                   ylabel='Number of products',
                   use_hover_tips=False,
                   background_fill_color='gainsboro',
                   toolbar_location='right',
                   ystart=0,
                   grid_line_color='white')
    data1 = xmatchCDS.data[data_cols[11]]
    data1 = data1[~np.isnan(data1)]
    hist1, edges1 = np.histogram(data1, bins=50)
    p1.build_histogram(top=hist1,
                       bottom=0,
                       left=edges1[:-1],
                       right=edges1[1:],
                       fill_color='navy',
                       fill_transparency=0.5,
                       line_color='white')
                       
    title2 = 'Median Separation (Sigma-clipped) of Point-to-Segment Cross-matched sources'
    p2 = HAPFigure(title=title2,
                   xlabel='Median Separation of Cross-matched sources (arcseconds)',
                   ylabel='Number of products',
                   use_hover_tips=False,
                   background_fill_color='gainsboro',
                   toolbar_location='right',
                   ystart=0,
                   grid_line_color='white')
    data2 = xmatchCDS.data[data_cols[12]]
    data2 = data2[~np.isnan(data2)]
    hist2, edges2 = np.histogram(data2, bins=50)
    p2.build_histogram(top=hist2,
                       bottom=0,
                       left=edges2[:-1],
                       right=edges2[1:],
                       fill_color='navy',
                       fill_transparency=0.5,
                       line_color='white')
                       
    title3 = 'Standard-deviation (sigma-clipped) of Separation of Point-to-Segment Cross-matched sources'
    p3 = HAPFigure(title='Number of Point-to-Segment Cross-matched sources',
                   xlabel='STD(Separation) of Cross-matched sources (arcseconds)',
                   ylabel='Number of products',
                   use_hover_tips=False,
                   background_fill_color='gainsboro',
                   toolbar_location='right',
                   ystart=0,
                   grid_line_color='white')
    data3 = xmatchCDS.data[data_cols[13]]
    data3 = data3[~np.isnan(data3)]
    hist3, edges3 = np.histogram(data3, bins=50)
    p3.build_histogram(top=hist3,
                       bottom=0,
                       left=edges3[:-1],
                       right=edges3[1:],
                       fill_color='navy',
                       fill_transparency=0.5,
                       line_color='white')

    # Display and save
    if display_plot:
        show(column(p0.fig, p1.fig, p2.fig, p3.fig))
    # Just save
    else:
        save(column(p0.fig, p1.fig, p2.fig, p3.fig))
    log.info("Output HTML graphic file {} has been written.\n".format(output))

    return output


def build_interfilter_crossmatch_plots(xm_df, display_plot, output_basename='svm_qa', log_level=logging.INFO):
    """"Generate plots to statistically quantify the quality of the alignment of filter-level HAP imagery
    products

    Parameters
    ----------
    xm_df : Pandas dataframe
        This dataframe contains all the columns relevant to the plots.

    output_basename : str
        String to use as the start of the filename for the output plot pages.

    display_plot : bool
        Option to display the plot in addition to writing out the file.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    -------
    output : str
        Name of HTML file where the plot was saved.
    """
    log.setLevel(log_level)
    # Setup the source of the data to be plotted so the axis variables can be
    # referenced by column name in the Pandas dataframe
    xm_cds = ColumnDataSource(xm_df)
    num_of_datasets = len(xm_cds.data['index'])
    log.info('Number of datasets: {}'.format(num_of_datasets))
    output_basename = "{}_interfilter_comparison_{}".format(output_basename,xm_df['gen_info.dataset'][0])

    if not output_basename.endswith('.html'):
        output = output_basename + '.html'
    else:
        output = output_basename
    # Set the output file immediately as advised by Bokeh.
    output_file(output)

    # Define plots
    plots = []
    div = Div(text="""
    <h1>Crossmatched interfilter comparison - reference residuals</h1>
    <h2>Crossmatch details</h2>""")
    plots.append(div)
    # plot #1: comp vs ref percent of all identified sources that were crossmatched
    tooltips_list = ['REF IMAGE', 'COMP IMAGE', 'TOTAL REF SOURCES', 'TOTAL COMP SOURCES', '# CROSSMATCHED']
    hover_columns = ['ref_imgname', 'comp_imgname', 'ref_catalog_length',
                     'comp_catalog_length', 'number_of_cross-matches']
    plot = make_scatter_plot(xm_cds,
                             'Percentage of all identified sources matched',
                             '% of all reference sources crossmatched',
                             '% of all comparison sources crossmatched',
                             'ref_crossmatch_percentage',
                             'comp_crossmatch_percentage',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 2: Non-clipped min: plot x vs y values for all filters
    div = Div(text="""<h2>Comparison - reference residual statistics</h2>
    <p>Reference image platescale: {} arcseconds per pixel</p>""".format(xm_df.ref_image_platescale[0]))
    plots.append(div)
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference non-clipped minimum separation',
                             'Minimum X separation (pixels)',
                             'Minimum Y separation (pixels)',
                             'x_non-clipped_min',
                             'y_non-clipped_min',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 3: Non-clipped max: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference non-clipped maximum separation',
                             'Maximum X separation (pixels)',
                             'Maximum Y separation (pixels)',
                             'x_non-clipped_max',
                             'y_non-clipped_max',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 4: Non-clipped mean: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference non-clipped mean separation',
                             'Mean X separation (pixels)',
                             'Mean Y separation (pixels)',
                             'x_non-clipped_mean',
                             'y_non-clipped_mean',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 5: Non-clipped median: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference non-clipped median separation',
                             'Median X separation (pixels)',
                             'Median Y separation (pixels)',
                             'x_non-clipped_median',
                             'y_non-clipped_median',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 6: Non-clipped standard deviation: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference separation non-clipped standard deviation',
                             'X separation SD (pixels)',
                             'Y separation SD (pixels)',
                             'x_non-clipped_rms',
                             'y_non-clipped_rms',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 7: 3x3 sigma-clipped mean: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference 3 x 3'+u'\u03c3'+'-clipped mean separation',
                             '3 x 3'+u'\u03c3'+'-clipped mean X separation (pixels)',
                             '3 x 3'+u'\u03c3'+'-clipped mean Y separation (pixels)',
                             'x_3x3_sigma-clipped_mean',
                             'y_3x3_sigma-clipped_mean',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 8: 3x3 sigma-clipped median: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference 3 x 3' + u'\u03c3' + '-clipped median separation',
                             '3 x 3' + u'\u03c3' + '-clipped median X separation (pixels)',
                             '3 x 3' + u'\u03c3' + '-clipped median Y separation (pixels)',
                             'x_3x3_sigma-clipped_median',
                             'y_3x3_sigma-clipped_median',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)

    # 9: 3x3 sigma-clipped standard deviation: plot x vs y values for all filters
    plot = make_scatter_plot(xm_cds,
                             'Comparision - reference separation 3 x 3' + u'\u03c3' + '-clipped standard deviation',
                             '3 x 3' + u'\u03c3' + '-clipped X SD (pixels)',
                             '3 x 3' + u'\u03c3' + '-clipped Y SD (pixels)',
                             'x_3x3_sigma-clipped_rms',
                             'y_3x3_sigma-clipped_rms',
                             tooltips_list,
                             hover_columns)
    plots.append(plot.fig)
    if display_plot:
        show(column(plots))  # display plot and save to html file
    else:
        save(column(plots))  # Just save
    log.info("Output HTML graphic file {} has been written.\n".format(output))

    # 10: delta_X vs delta_Y vector plot for each cross-matched filter and/or x vs delta_x, x vs. delta_y, y
    # vs. delta_x, y vs. delta_y quad plot each cross-matched filter
    # Set the output file immediately as advised by Bokeh.

    for line_index in xm_df.index:
        xm_df2 = xm_df[xm_df['gen_info.dataframe_index'] == line_index]

        resid_output = output.replace(".html", "_{}_{}_{}_residuals.html".format(xm_df2['gen_info.instrument'][0].lower(),
                                                                                 xm_df2['gen_info.detector'][0].lower(),
                                                                                 xm_df2['gen_info.filter'][0].lower()))
        output_file(resid_output)
        plots = []
        div = Div(text="""
        <h1>Position vs. Crossmatched interfilter comparison - reference residual values</h1>
        <p>Comparison image: {}<br/>Reference image: {}<br/>
        Reference image platescale: {} arcseconds per pixel</p>""".format(xm_df2['comp_imgname'][line_index],
                                                                          xm_df2['ref_imgname'][line_index],
                                                                          xm_df.ref_image_platescale[0]))
        plots.append(div)
        xm_cds = ColumnDataSource(xm_df2)
        cds2_dict = {"x": xm_cds.data["ref_catalog.xcentroid_ref"][0],
                     "y": xm_cds.data["ref_catalog.ycentroid_ref"][0],
                     "dx": xm_cds.data["comp-ref_x_seperation"][0],
                     "dy": xm_cds.data["comp-ref_y_seperation"][0]}
        residsCDS = ColumnDataSource(cds2_dict)
        quad_plot = make_quad_scatter_plot(residsCDS)
        plots += quad_plot

        plot = build_vector_plot(residsCDS, title='Vector plot of Comparison - Reference residuals',
                                 x_label='X (pixels)', y_label='Y (pixels)', color='blue')
        plots.append(plot.fig)
        if display_plot:
            show(column(plots))  # display plot and save to html file
        else:
            save(column(plots))  # Just save
        log.info("Output HTML graphic file {} has been written.\n".format(resid_output))



    return output
# -----------------------------------------------------------------------------
# Functions for generating the photometry plots
#


def generate_photometry_graphic(phot_dataDF, output_base_filename='', display_plot=False,
                                log_level=logutil.logging.INFO):
    """Generate the graphics associated with this particular type of data.

    Parameters
    ==========
    phot_dataDF : Pandas dataframe
    Dataframe consisting of the Magnitude statistics of mean, std, and median for Aperture 1
    and Aperture 2

    output_base_filename : str
    Base name for the HMTL file generated by Bokeh.

    display_plot : bool
        Option to display the plot to the screen
        Default: False

    log_level : int
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    =======
    Nothing

    HTML file is generated and written to the current directory.
    """
    log.setLevel(log_level)

    # Set the output file immediately as advised by Bokeh.
    if output_base_filename == '':
        output_base_filename = '"svm_qa_photometry'
    else:
        output_base_filename = '{}_svm_qa_photometry'.format(output_base_filename)
    output_file(output_base_filename + '.html')

    # Compute some statistics to report on plot
    mean_dMagAp1mean = phot_dataDF['Ap1 Mean Differences'].mean()
    mean_dMagAp1median = phot_dataDF['Ap1 Median Differences'].mean()

    mean_dMagAp2mean = phot_dataDF['Ap2 Mean Differences'].mean()
    mean_dMagAp2median = phot_dataDF['Ap2 Median Differences'].mean()

    # Setup the source of the data to be plotted so the axis variables can be
    # referenced by column name in the Pandas dataframe
    sourceCDS = ColumnDataSource(phot_dataDF)
    num_of_datasets = len(phot_dataDF.index)
    log.info('Number of datasets: {}.\n'.format(num_of_datasets))

    info_text = 'Number of datasets: ' + str(num_of_datasets)

    # Define some tooltips
    photometry_tooltips = [("Mean", "@{Ap1 Mean Differences}"),
                           ("StdDev", "@{Ap1 Standard Deviation}"),
                           ("Median", "@{Ap1 Median Differences}")]

    # Aperture 1
    p0 = HAPFigure(title='Mean Aperture 1 Magnitude Differences (Point - Segment)',
                   x_label='Index     ' + info_text,
                   y_label='Difference (ABMag)',
                   hover_tips=photometry_tooltips)
    p0.build_glyph('circle',
                   x='x_index',
                   y='Ap1 Mean Differences',
                   sourceCDS=sourceCDS,
                   glyph_color='colormap',
                   legend_group='inst_det',
                   name="ap1mean")
    stat_text = ('<DeltaMagAp1_Mean>: {:6.2f}'.format(mean_dMagAp1mean))
    stat_label = Label(x=20, y=20, x_units='screen', y_units='screen', text=stat_text)
    p0.fig.add_layout(stat_label)

    p1 = HAPFigure(title='Median Aperture 1 Magnitude Differences (Point - Segment)',
                   x_label='Index     ' + info_text,
                   y_label='Difference (ABMag)',
                   x_range=p0.fig.x_range,
                   y_range=p0.fig.y_range,
                   hover_tips=photometry_tooltips)
    p1.build_glyph('circle',
                   x='x_index',
                   y='Ap1 Median Differences',
                   sourceCDS=sourceCDS,
                   glyph_color='colormap',
                   name='ap1median')
    stat_text = ('<DeltaMagAp1_Median>: {:6.2f}'.format(mean_dMagAp1median))
    stat_label = Label(x=20, y=20, x_units='screen', y_units='screen', text=stat_text)
    p1.fig.add_layout(stat_label)

    # Aperture 2
    photometry_tooltips = [("Dataset", "@dataset"),
                           ("Mean", "@{Ap2 Mean Differences}"),
                           ("StdDev", "@{Ap2 Standard Deviation}"),
                           ("Median", "@{Ap2 Median Differences}")]

    # Add the glyphs
    # Aperture 2
    p2 = HAPFigure(title='Mean Aperture 2 Magnitude Differences (Point - Segment)',
                   x_label='Index     ' + info_text,
                   y_label='Difference (ABMag)',
                   hover_tips=photometry_tooltips)
    p2.build_glyph('circle',
                   x='x_index',
                   y='Ap2 Mean Differences',
                   sourceCDS=sourceCDS,
                   glyph_color='colormap',
                   legend_group='inst_det',
                   name='ap2mean')
    stat_text = ('<DeltaMagAp2_Mean>: {:6.2f}'.format(mean_dMagAp2mean))
    stat_label = Label(x=20, y=20, x_units='screen', y_units='screen', text=stat_text)
    p2.fig.add_layout(stat_label)

    p3 = HAPFigure(title='Median Aperture 2 Magnitude Differences (Point - Segment)',
                   x_label='Index     ' + info_text,
                   y_label='Difference (ABMag)',
                   x_range=p2.fig.x_range,
                   y_range=p2.fig.y_range,
                   hover_tips=photometry_tooltips)
    p3.build_glyph('circle',
                   x='x_index',
                   y='Ap2 Median Differences',
                   sourceCDS=sourceCDS,
                   glyph_color='colormap',
                   name='ap2median')
    stat_text = ('<DeltaMagAp2_Median>: {:6.2f}'.format(mean_dMagAp2median))
    stat_label = Label(x=20, y=20, x_units='screen', y_units='screen', text=stat_text)
    p3.fig.add_layout(stat_label)

    grid = gridplot([[p0.fig, p1.fig], [p2.fig, p3.fig]], plot_width=500, plot_height=500)
    # Display and save
    if display_plot:
        show(grid)
    # Just save
    else:
        save(grid)
    log.info("Output HTML graphic file {} has been written.\n".format(output_base_filename + ".html"))


def photometry_graphics_driver(storage_filename, output_base_filename='', display_plot=False,
                               log_level=logutil.logging.NOTSET):
    """Driver to load the data from the storage file and generate the graphics.

    Parameters
    ==========
    storage_filename : str
    Name of the storage file for the Pandas dataframe created by the harvester.

    output_base_filename : str
    Base name for the HMTL file generated by Bokeh.

    display_plot : bool
        Option to display the plot in addition to writing out the file.
        Default is False.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    =======
    Nothing
    """
    log.setLevel(log_level)

    # Dictionary mapping the original dataframe column name to their new simple names
    phot_columns = {'Statistics_MagAp1.Delta_MagAp1.Mean': 'Ap1 Mean Differences',
                    'Statistics_MagAp1.Delta_MagAp1.StdDev': 'Ap1 Standard Deviation',
                    'Statistics_MagAp1.Delta_MagAp1.Median': 'Ap1 Median Differences',
                    'Statistics_MagAp2.Delta_MagAp2.Mean': 'Ap2 Mean Differences',
                    'Statistics_MagAp2.Delta_MagAp2.StdDev': 'Ap2 Standard Deviation',
                    'Statistics_MagAp2.Delta_MagAp2.Median': 'Ap2 Median Differences'}

    # Retrieve the dataframe
    log.info('Photometry_graphics. Retrieve Pandas dataframe from file {}.\n'.format(storage_filename))
    phot_dataDF = get_pandas_data(storage_filename, phot_columns.keys())

    # Rename the columns to abbreviated text as the graph titles further
    # document the information.
    for old_col_name, new_col_name in phot_columns.items():
        phot_dataDF.rename(columns={old_col_name: new_col_name}, inplace=True)

    # Generate a general index array and add it to the dataframe
    x_index = list(range(0, len(phot_dataDF.index)))
    phot_dataDF['x_index'] = x_index
    x_index.clear()

    # Generate the photometric graphic
    generate_photometry_graphic(phot_dataDF, output_base_filename, display_plot, log_level)

# -----------------------------------------------------------------------------
# Number of detected sources in Point in Segment catalogs
#


def generate_nsources_graphic(dataDF, output_base_filename='', display_plot=False,
                              log_level=logutil.logging.INFO):
    """Generate plot displaying the difference in the number of sources detected in the Point and Segment
    catalogs.

    Parameters
    ==========
    dataDF : Pandas dataframe
        A subset of the input Dataframe consisting of the number of sources detected
        in the output catalogs (point and segment), as well as the default information
        added by the pandas_utils.py module.

    output_base_filename : str
        Base name for the HMTL file generated by Bokeh.

    display_plot : bool
        Option to display the plot to the screen
        Default: False

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default: 20 or 'info'.

    Returns
    =======
    Nothing

    HTML file is generated and written to the current directory.
    """
    log.setLevel(log_level)

    # Set the output file immediately as advised by Bokeh.
    if output_base_filename == '':
        output_base_filename = '"svm_qa_cat_nsources'
    else:
        output_base_filename = '{}_svm_qa_cat_nsources'.format(output_base_filename)
    output_file(output_base_filename + '.html')

    # Set the output file immediately as advised by Bokeh.
    output_file(output_base_filename + '.html')

    num_of_input_files = len(dataDF.index)
    log.info('Number of individual catalog files: {}'.format(num_of_input_files))

    # Generate a general index array and add it to the dataframe
    x_index = list(range(0, len(dataDF.index)))
    dataDF['x_index'] = x_index
    x_index.clear()

    # Setup the source of the data to be plotted so the axis variables can be
    # referenced by column name in the Pandas dataframe
    cds = ColumnDataSource(dataDF)

    # Instantiate the figure objects
    text = 'Difference in Number of Detected Sources between Catalogs (Point - Segment)'
    p1 = HAPFigure(title=text,
                   x_label='Index',
                   y_label='Difference (counts)')
    p1.build_glyph('circle',
                   x='x_index',
                   y='nsrcs_difference',
                   sourceCDS=cds,
                   glyph_color='colormap',
                   legend_group='inst_det')

    # Display and save
    if display_plot:
        show(p1.fig)
    # Just save
    else:
        save(p1.fig)
    log.info("Output HTML graphic file {} has been written.\n".format(output_base_filename + ".html"))


def nsources_graphics_driver(storage_filename, output_base_filename='', display_plot=False,
                             log_level=logutil.logging.INFO):
    """Driver to load the data from the storage file and generate the graphics.

    This particular is to display the differences in the number of detected sources documented
    in the Point and Segment catalogs (*.ecsv).

    Parameters
    ==========
    storage_filename : str
        Name of the storage file for the Pandas dataframe created by the harvester.

    output_base_filename : str
        Base name for the HMTL file generated by Bokeh.

    display_plot : bool, optional
        Option to display the plot in addition to writing out the file.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    =======
    Nothing
    """
    log.setLevel(log_level)

    # Dictionary of Pandas dataframe column names mapped to simple names
    nsources_columns = {'number_of_sources.point': 'nsrcs_point',
                        'number_of_sources.segment': 'nsrcs_segment'}

    # Retrieve the relevant dataframe columns
    log.info('Retrieve Pandas dataframe from file {}.\n'.format(storage_filename))
    dataDF = get_pandas_data(storage_filename, nsources_columns.keys())

    # Rename the columns to abbreviated text as the graph titles further
    # document the information.
    for old_col_name, new_col_name in nsources_columns.items():
        dataDF.rename(columns={old_col_name: new_col_name}, inplace=True)

    # Compute the differences between the number of sources and add the computed
    # column to the dataframe
    dataDF['nsrcs_difference'] = dataDF['nsrcs_point'] - dataDF['nsrcs_segment']

    # Generate the graphic
    generate_nsources_graphic(dataDF, output_base_filename, display_plot, log_level)

# -----------------------------------------------------------------------------
# WCS graphic - comparison of crpix, crval, scale, and orientation
#


def generate_wcs_graphic(wcs_dataDF, wcs_columns, output_base_filename='', display_plot=False,
                         log_level=logutil.logging.NOTSET):
    """Generate the graphics associated with this particular type of data.

    Parameters
    ==========
    wcs_dataDF : Pandas dataframe
        A subset of the input Dataframe consisting only of the wcs_columns
        and Aperture 2

    wcs_columns : dict
        Dictionary of original column names (keys) as stored in the Pandas dataframe
        and the corresponding simple name (values) which is often more practical for use.

    output_base_filename : str
        Base name for the HMTL file generated by Bokeh.

    display_plot : bool, optional
        Option to display the plot to the screen
        Default: False

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default: 20 or 'info'.

    Returns
    =======
    Nothing

    HTML file is generated and written to the current directory.
    """
    log.setLevel(log_level)

    # Set the output file immediately as advised by Bokeh.
    if output_base_filename == '':
        output_base_filename = 'svm_qa_wcs'
    else:
        output_base_filename = '{}_svm_qa_wcs'.format(output_base_filename)
    output_file(output_base_filename + '.html')

    # Setup the source of the data to be plotted so the axis variables can be
    # referenced by column name in the Pandas dataframe
    sourceCDS = ColumnDataSource(wcs_dataDF)
    num_of_datasets = len(wcs_dataDF.index)
    log.info('Number of datasets: {}'.format(num_of_datasets))

    # Figure 1 delta_crpix1 vs delta_crpix2
    # Figure 2 delta_crval1 vs delta_crval2
    # Figure 3 delta_scale vs delta_orientation
    wcs_type_names = ['default', 'apriori', 'aposteriori']
    wcs_components = []
    alt_wcs_names = []
    for wcs_type in wcs_type_names:
        wcs_components.append('del_{}_wcsname'.format(wcs_type))
        alt_wcs_names.append('alt_{}_wcsname'.format(wcs_type))

    # There are three distinct figures in each row and one row
    # for each wcs_type_names
    fig_list = []
    for i, wcs_component in enumerate(wcs_components):
        if wcs_component in wcs_dataDF.columns:
            slist = wcs_component.rsplit('_')
            text = 'WCS Difference: Primary - '+slist[1].capitalize()

            # Build the hover tooltips on-the-fly as the altername WCS is a variable
            alt_value = '"@{}"'.format(alt_wcs_names[i])
            wcs_tips = [("Primary", "@prim_wcsname"),
                        ("Alternate", alt_value)]

            fig = HAPFigure(title=text,
                            x_label='Delta CRPIX1 (pixels)',
                            y_label='Delta CRPIX2 (pixels)',
                            hover_tips=wcs_tips)
            fig.build_glyph('circle',
                            x='del_{}_crpix1'.format(slist[1]),
                            y='del_{}_crpix2'.format(slist[1]),
                            sourceCDS=sourceCDS,
                            glyph_color='colormap',
                            legend_group='inst_det')
            fig_list.append(fig)

            fig = HAPFigure(title=text,
                            x_label='Delta CRVAL1 (degrees)',
                            y_label='Delta CRVAL2 (degrees)',
                            hover_tips=wcs_tips)
            fig.build_glyph('circle',
                            x='del_{}_crval1'.format(slist[1]),
                            y='del_{}_crval2'.format(slist[1]),
                            sourceCDS=sourceCDS,
                            glyph_color='colormap',
                            legend_group='inst_det')
            fig_list.append(fig)

            fig = HAPFigure(title=text,
                            x_label='Delta Scale (pixels/arcseconds)',
                            y_label='Delta Orientation (degrees)',
                            hover_tips=wcs_tips)
            fig.build_glyph('circle',
                            x='del_{}_scale'.format(slist[1]),
                            y='del_{}_orient'.format(slist[1]),
                            sourceCDS=sourceCDS,
                            glyph_color='colormap',
                            legend_group='inst_det')
            fig_list.append(fig)

    # Create the the HTML output and optionally display the plot
    grid = gridplot([[fig_list[0].fig, fig_list[1].fig, fig_list[2].fig],
                    [fig_list[3].fig, fig_list[4].fig, fig_list[5].fig],
                    [fig_list[6].fig, fig_list[7].fig, fig_list[8].fig]],
                    plot_width=500, plot_height=500)

    # Display and save
    if display_plot:
        show(grid)
    # Just save
    else:
        save(grid)
    log.info("Output HTML graphic file {} has been written.\n".format(output_base_filename + ".html"))


def wcs_graphics_driver(storage_filename, output_base_filename='', display_plot=False,
                        log_level=logutil.logging.INFO):
    """Driver to load the data from the storage file and generate the graphics.

    Parameters
    ==========
    storage_filename : str
        Name of the storage file for the Pandas dataframe created by the harvester.

    output_base_filename : str, optional
        Base name for the HMTL file generated by Bokeh.

    display_plot : bool, optional
        Option to display the plot in addition to writing out the file.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default value is 20, or 'info'.

    Returns
    =======
    Nothing
    """
    log.setLevel(log_level)

    # Observations taken after Oct 2017 will most likely NOT have any apriori
    # solutions since they were observed using GAIA-based coordinates for the
    # guide stars by default.  In a sense, the default IDC_<rootname> WCS is
    # the apriori solution.
    wcs_columns = {'PrimaryWCS.primary_wcsname': 'prim_wcsname',
                   'PrimaryWCS.crpix1': 'prim_crpix1',
                   'PrimaryWCS.crpix2': 'prim_crpix2',
                   'PrimaryWCS.crval1': 'prim_crval1',
                   'PrimaryWCS.crval2': 'prim_crval2',
                   'PrimaryWCS.scale': 'prim_scale',
                   'PrimaryWCS.orientation': 'prim_orient',
                   'PrimaryWCS.exposure': 'prim_exponame',
                   'AlternateWCS_default.alt_wcsname': 'alt_default_wcsname',
                   'AlternateWCS_default.crpix1': 'alt_default_crpix1',
                   'AlternateWCS_default.crpix2': 'alt_default_crpix2',
                   'AlternateWCS_default.crval1': 'alt_default_crval1',
                   'AlternateWCS_default.crval2': 'alt_default_crval2',
                   'AlternateWCS_default.scale': 'alt_default_scale',
                   'AlternateWCS_default.orientation': 'alt_default_orient',
                   'AlternateWCS_default.exposure': 'alt_default_exponame',
                   'DeltaWCS_default.delta_wcsname': 'del_default_wcsname',
                   'DeltaWCS_default.d_crpix1': 'del_default_crpix1',
                   'DeltaWCS_default.d_crpix2': 'del_default_crpix2',
                   'DeltaWCS_default.d_crval1': 'del_default_crval1',
                   'DeltaWCS_default.d_crval2': 'del_default_crval2',
                   'DeltaWCS_default.d_scale': 'del_default_scale',
                   'DeltaWCS_default.d_orientation': 'del_default_orient',
                   'DeltaWCS_default.exposure': 'del_default_exponame',
                   'AlternateWCS_apriori.alt_wcsname': 'alt_apriori_wcsname',
                   'AlternateWCS_apriori.crpix1': 'alt_apriori_crpix1',
                   'AlternateWCS_apriori.crpix2': 'alt_apriori_crpix2',
                   'AlternateWCS_apriori.crval1': 'alt_apriori_crval1',
                   'AlternateWCS_apriori.crval2': 'alt_apriori_crval2',
                   'AlternateWCS_apriori.scale': 'alt_apriori_scale',
                   'AlternateWCS_apriori.orientation': 'alt_apriori_orient',
                   'AlternateWCS_apriori.exposure': 'alt_apriori_exponame',
                   'DeltaWCS_apriori.delta_wcsname': 'del_apriori_wcsname',
                   'DeltaWCS_apriori.d_crpix1': 'del_apriori_crpix1',
                   'DeltaWCS_apriori.d_crpix2': 'del_apriori_crpix2',
                   'DeltaWCS_apriori.d_crval1': 'del_apriori_crval1',
                   'DeltaWCS_apriori.d_crval2': 'del_apriori_crval2',
                   'DeltaWCS_apriori.d_scale': 'del_apriori_scale',
                   'DeltaWCS_apriori.d_orientation': 'del_apriori_orient',
                   'DeltaWCS_apriori.exposure': 'del_apriori_exponame',
                   'AlternateWCS_aposteriori.alt_wcsname': 'alt_aposteriori_wcsname',
                   'AlternateWCS_aposteriori.crpix1': 'alt_aposteriori_crpix1',
                   'AlternateWCS_aposteriori.crpix2': 'alt_aposteriori_crpix2',
                   'AlternateWCS_aposteriori.crval1': 'alt_aposteriori_crval1',
                   'AlternateWCS_aposteriori.crval2': 'alt_aposteriori_crval2',
                   'AlternateWCS_aposteriori.scale': 'alt_aposteriori_scale',
                   'AlternateWCS_aposteriori.orientation': 'alt_aposteriori_orient',
                   'AlternateWCS_aposteriori.exposure': 'alt_aposteriori_exponame',
                   'DeltaWCS_aposteriori.delta_wcsname': 'del_aposteriori_wcsname',
                   'DeltaWCS_aposteriori.d_crpix1': 'del_aposteriori_crpix1',
                   'DeltaWCS_aposteriori.d_crpix2': 'del_aposteriori_crpix2',
                   'DeltaWCS_aposteriori.d_crval1': 'del_aposteriori_crval1',
                   'DeltaWCS_aposteriori.d_crval2': 'del_aposteriori_crval2',
                   'DeltaWCS_aposteriori.d_scale': 'del_aposteriori_scale',
                   'DeltaWCS_aposteriori.d_orientation': 'del_aposteriori_orient',
                   'DeltaWCS_aposteriori.exposure': 'del_aposteriori_exponame'}

    # Retrieve the relevant dataframe columns
    log.info('Retrieve Pandas dataframe from file {}.\n'.format(storage_filename))
    wcs_dataDF = get_wcs_data(storage_filename, wcs_columns, log_level)

    # Rename the columns to abbreviated text as the graph titles further
    # document the information.
    for old_col_name, new_col_name in wcs_columns.items():
        wcs_dataDF.rename(columns={old_col_name: new_col_name}, inplace=True)

    # Generate the WCS graphic
    generate_wcs_graphic(wcs_dataDF, wcs_columns, output_base_filename, display_plot, log_level)

# -----------------------------------------------------------------------------
# General Utility functions for plotting
#


def get_wcs_data(storage_filename, wcs_columns, log_level=logutil.logging.NOTSET):
    """Load the harvested data, stored in a storage file, into local arrays.

    Parameters
    ==========
    storage_filename : str
        Name of the storage file for the Pandas dataframe created by the harvester.

    wcs_columns : dict
        Dictionary of original column names (keys) as stored in the Pandas dataframe
        and the corresponding simple name (values) which is often more practical for use.

    log_level : int, optional
        The desired level of verboseness in the log statements displayed on the screen and written to the
        .log file. Default: 20 or 'info'.

    Returns
    =======
    wcs_dataDF : Pandas dataframe
        Dataframe which is a subset of the input Pandas dataframe which
        consists of only the requested columns and rows, as well as any columns provided
        by pandas_utils for free.

    Note: This routine is different from the nominal get_pandas_data() in that it tries to
    compensate in case on the of requested columns (apriori or aposteriori) is missing.
    """
    log.setLevel(log_level)

    # Instantiate a Pandas Dataframe Reader (lazy instantiation)
    df_handle = PandasDFReader(storage_filename, log_level=log_level)

    # Get the relevant column data
    wcs_column_type = ['apriori', 'aposteriori']
    windex = -1
    wcs_dataDF = pd.DataFrame()
    while wcs_dataDF.empty:
        wcs_dataDF = df_handle.get_columns_HDF5(wcs_columns.keys())

        # If no dataframe were returned, there was a KeyError because columns were
        # not present in the original dataframe versus the columns contained NaNs.
        entries_to_remove = []
        if wcs_dataDF.empty and windex < len(wcs_column_type) - 1:
            log.info("Columns are missing from the data file {}. Remove some requested column names\n"
                     "from the list and try again.\n".format(storage_filename))

            # Identify missing columns and remove them from a copy of the original dictionary
            windex += 1
            for key in wcs_columns:
                if re.match(r'.+{}.+'.format(wcs_column_type[windex]), key):
                    entries_to_remove.append(key)

            for key in entries_to_remove:
                wcs_columns.pop(key, None)

        elif wcs_dataDF.empty:
            log.critical("Critical columns not found in storage Pandas dataframe: {}.\n".format(
                storage_filename))
            sys.exit(1)

    log.info("WCS data has been retrieved from the storage Pandas dataframe: {}.\n".format(storage_filename))

    return wcs_dataDF


def make_quad_scatter_plot(residsCDS):
    """Create x vs dx, x vs dy, y vs. dx and y vs. dy scatter plots in a 2x2 grid

    Parameters
    ----------
    residsCDS : bokeh.models.sources.ColumnDataSource object
        bokeh ColumnDataSoruce containing x, y, dx and dy values to plot

    Returns
    -------
    Two-element list of bokeh.models.layouts.Row objects
    """
    if len(residsCDS.data['x']) < 2:
        return None

    p1 = HAPFigure(title='X vs DX',
                   x_label="X (pixels)",
                   y_label='Delta[X] (pixels)',
                   use_hover_tips=False)
    p1.build_glyph('circle',
                   x='x',
                   y='dx',
                   sourceCDS=residsCDS)

    p2 = HAPFigure(title='X vs DY',
                   x_label="X (pixels)",
                   y_label='Delta[Y] (pixels)',
                   use_hover_tips=False)
    p2.build_glyph('circle',
                   x='x',
                   y='dy',
                   sourceCDS=residsCDS)

    row1 = row(p1.fig, p2.fig)

    p3 = HAPFigure(title='Y vs DX',
                   x_label="Y (pixels)",
                   y_label='Delta[X] (pixels)',
                   use_hover_tips=False)
    p3.build_glyph('circle',
                   x='y',
                   y='dx',
                   sourceCDS=residsCDS)

    p4 = HAPFigure(title='Y vs DY',
                   x_label="Y (pixels)",
                   y_label='Delta[Y] (pixels)',
                   use_hover_tips=False)
    p4.build_glyph('circle',
                   x='y',
                   y='dy',
                   sourceCDS=residsCDS)

    row2 = row(p3.fig, p4.fig)

    return [row1, row2]


def make_scatter_plot(xm_cds, title, x_label, y_label, x_data_colname, y_data_colname, tooltips_list,
                      hover_columns):
    """create a bokeh scatter plot with hovertools

    Parameters
    ----------
    cds : Pandas dataframe
        This dataframe contains all the columns relevant to the plots.

    title : str
        plot title

    x_label : str
        X-axis label

    y_label : str
        Y-axis label

    x_data_colname : str
        **cds** data element to plot on the X axis

     y_data_colname : str
        **cds** data element to plot on the y axis

    tooltips_list : list
        list of hovertool element titles

    hover_columns : list
        list of the data column names that will be queried to populate the hovertool information

    Returns
    -------
    scatter_plot :

    """
    hover_tips = build_tooltips(tooltips_list, hover_columns, list(range(0, len(hover_columns))))
    scatterplot = HAPFigure(title=title, x_label=x_label, y_label=y_label, hover_tips=hover_tips)
    scatterplot.build_glyph('circle',
                            x=x_data_colname,
                            y=y_data_colname,
                            sourceCDS=xm_cds,
                            glyph_color='colormap',
                            legend_group='inst_det')
    return scatterplot


if __name__ == "__main__":
    build_svm_plots(sys.argv[1], output_basename='', display_plot=True, log_level=logging.DEBUG)
