#!/usr/bin/python

import getopt
import hashlib
import inspect
import json
import logging
import os
import os.path
import pickle
import re
import sqlite3
import stat
import subprocess
import sys
import time

import isi.misc as misc
from isi.app.lib import safelockfile

PROGPATH = sys.argv[0]
PROGNAME = os.path.basename(PROGPATH)
PROGLOCN = os.path.dirname(PROGPATH)
REVISION = "$Revision: 293117 $"
VERSION = 1
TITLE = "Drive Support Package Tool"

##############################################################################
##   ____                                     _       
##  / ___|___  _ __ ___  _ __ ___   ___ _ __ | |_ ___ 
## | |   / _ \| '_ ` _ \| '_ ` _ \ / _ \ '_ \| __/ __|
## | |__| (_) | | | | | | | | | | |  __/ | | | |_\__ \
##  \____\___/|_| |_| |_|_| |_| |_|\___|_| |_|\__|___/
##                                                    
##############################################################################

# For DSP isi pkg install, the original thought was to use isi_for_array to
# have each node run this script locally with "--syncifslocal" command to sync
# local node files with the installed ifs files.  However, isk pkg actually
# runs the entire script in parallel on all nodes (including creating staging
# memory disks on each node).  Therefore, there is no need for additional
# "isi_for_array" commands by this script.  We still do a InstallFiles.sync()
# at end of install, with a direct sync_ifs_local() call, as an attempt to
# keep the entire cluster consistent -- particularly for any non-standard
# cases (such as nodes joining the cluster, or --forced_local installs).

# Originally a database was created, on /ifs/.ifsvar/db/dsp, to track install
# history -- to be used with both "Rollback" and "Override" features.
# However, we decided NOT to implement any rollback functionality, which was
# the primary driver for a cluster-wide database of DSP install history.
# Further, overrides are more easily handled as simple files, resident in the
# same directories as the files they impact.  Finally, updating a common ifs
# database has its own set of problems -- isi pkg install runs the install
# script in parallel on ALL nodes, so we'd have to lock the (SQLite) database,
# since it is not parallel access thread safe.  We'd have to do this locking
# in some sort of synchronized manner, across all node installs, which is
# non-trivial.  We'd have to deal with any "forced_local" non-ifs installs
# and their own histories (either using a local database, which would then
# need to be later synchronized/merged with the ifs database when the node
# re-joins the cluster -- similar to celog db.  Or we would skip history
# tracking altogether for local non-ifs installs.
# Currently, the database is DISABLED (DATABASE_TRACKER = None).


##############################################################################
##  ____       _                 _                 _   _____ _                 
## | __ )  ___| |__   __ ___   _(_) ___  _ __ __ _| | |  ___| | __ _  __ _ ___ 
## |  _ \ / _ \ '_ \ / _` \ \ / / |/ _ \| '__/ _` | | | |_  | |/ _` |/ _` / __|
## | |_) |  __/ | | | (_| |\ V /| | (_) | | | (_| | | |  _| | | (_| | (_| \__ \
## |____/ \___|_| |_|\__,_| \_/ |_|\___/|_|  \__,_|_| |_|   |_|\__,_|\__, |___/
##                                                                   |___/     
##############################################################################

DELETE_STAGING_FILES = False

# By default, report to console as well as log file
# For isi pkg install, clear this using --quiet flag
VERBOSE = True

# DATABASE_TRACKER "build" flag is None, "ifs", or "local"
DATABASE_TRACKER = None

# If installing a DSP package which was already installed,
# should we update the corresponding database entry timestamp?
DATABASE_UPDATE_PKG_INSTALL_TIME = True

# Should --selected display filepath (False) or contents (True)
DISPLAY_SELECTED_FILE_CONTENTS = True

# Warn or Error?, if ifs unmounted (for InstalledFiles; not for InstallFiles)
WARN_IF_IFS_UNMOUNTED = True

##############################################################################
##  ____        __ _       _ _   _                 
## |  _ \  ___ / _(_)_ __ (_) |_(_) ___  _ __  ___ 
## | | | |/ _ \ |_| | '_ \| | __| |/ _ \| '_ \/ __|
## | |_| |  __/  _| | | | | | |_| | (_) | | | \__ \
## |____/ \___|_| |_|_| |_|_|\__|_|\___/|_| |_|___/
##                                                 
##############################################################################

# Defined by isi pkg
LOCAL_FLAG = 'forced_local'

IFS_DSP_DIRECTORY = '/ifs/.ifsvar/db/dsp'
IFS_DSP_LOCATION = lambda loc: os.path.join(IFS_DSP_DIRECTORY, loc)

# LOCAL alternative can be used instead of IFS for --forced_local, etc,
# similar location to other "local" Isilon dirs (pkg, celog, etc).
LOCAL_DSP_DIRECTORY = '/var/db/dsp'
LOCAL_DSP_LOCATION = lambda loc: os.path.join(LOCAL_DSP_DIRECTORY, loc)

# Could make this LOCATIONS lookup dict dependent on version.
# [contents]/config
#              ./drives/*
# [contents]/firmware
#              ./$VENDOR/[$MODEL]/*

# LOCATIONS values:
# String: Full path to node install location
# List  : Dual path install locations, "primary" and "secondary"

# Initial design is that the primary location is on the ifs partition,
# "if available", and the secondary location is the local node location,
# kept in sync (identical) with the primary ifs location "if possible".
# Install:
#   Install to primary (ifs) location, when possible.
#   Install to secondary (local) location, independently (direct).
#   After install: Sync (compare, copy/merge) ifs and local locations.
# Inventory (file listings, file searches for matches, etc):
#   Get both primary and (local) secondary listing, when possible.
#   Find and report any inconsistencies.
#   Resolve any inconsistencies (Add, only, where missing), via Sync.
# Modifications (Read/Modify/Write of internal files [2]):
#   Read both primary and (local) secondary contents, when possible.
#   Find and report any inconsistencies.
#   Resolve any inconsistencies with clearly defined rules (Sync).
#   Commit modifications to primary, when possible.
#   Sync to secondary location, ON ALL NODES [1].

# Note [1]: Sync (copy) from primary to secondary locations requires
# cluster wide operations, presumably using "isi_for_array" or similar,
# because we want this sync to go to ALL (local) node secondary locations.
# A simpler approach is possible -- delay sync with ifs until the data is
# actually read, locally, performaing local file sync mods at that time.
# This way no "isi_for_array" is needed: Sync is done at Read time, not at
# modify Write time.  HOWEVER, a drawback to this simpler approach is that
# if "something" happens between modify (Write to ifs) and a later read
# by a remote node, such as that node being removed from the cluster,
# then no ifs sync is possible, and the node has "stale" data.  This is
# an unusual non-standard case, but needs to be considered.

# Note [2]: "Internal files" refers to "hidden" files providing
# functionality, such as:
#  * Config File Version Overrides
#  * Config File Custom Data Overrides
#  * Config File Selection State (history)
# The install history DATABASE_FILE, if kept on ifs (ref: DATABASE_TRACKER),
# is NOT considered one of the primary/secondary synced LOCATIONS files.
# If ifs is unavailable, we are simply not updating the (ifs) database.

# Firmware files do not have a secondary location.
# If a forced_local install is done, simply keep the pkg installed
# and use the staging location (passed to tool as "staging" var).
# We COULD add a secondary location here, an indicator to use
# (check) this TBD staging var location, but with caveat that it
# is NOT to be synced with primary. XXX TBD Implementation Details.

LOCATIONS = {
    '.snap': None,
    'config':
    { 'drives':
      [ IFS_DSP_LOCATION('hw/config/drives'),
        '/etc/ifs/hw/config/drives' ]
    },
    'firmware': IFS_DSP_LOCATION('firmware'),
    }

LOG_FILE = '/var/log/isi_dsptool.log'
LOG_LEVEL = logging.INFO
LOG_FORMAT = "%(asctime)s %(levelname)-8s: %(message)s"
LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S"

DATABASE_FILE = {
    'ifs': IFS_DSP_LOCATION('history.db'),
    'local': LOCAL_DSP_LOCATION('history.db'),
    }.get(DATABASE_TRACKER, 'None')

OVERRIDE_VERS_FILENAME = '.overrides'
OVERRIDE_DATA_FILENAME = '.custom'
SELECT_STATE_FILENAME = '.state'
IFS_SYNC_LOCK_FILENAME = '.synclock'

# INSTALLED_FILES_FILTER_LIST is used to filter out files that we do NOT
# want to sync or copy between ifs and local. I.e, "filter these out from
# the installed files list that we sync..."

INSTALLED_FILES_FILTER_LIST = [
    OVERRIDE_VERS_FILENAME,
    OVERRIDE_DATA_FILENAME,
    SELECT_STATE_FILENAME,
    IFS_SYNC_LOCK_FILENAME,
    ]

SORT_TYPES = ["name", "time", "size", "version"]
DEFAULT_SORT = "name"

HASH_TYPES = ["none", "md5", "sha1", "sha256", "rmd160"]
DEFAULT_HASH = "none"

JSON_FILE_REGEX       = re.compile('^(.+)\.json(\.[0-9]+)?$')
JSON_DSP_FILE_REGEX   = re.compile('^(.+)\.json(\.[0-9]+)$')
JSON_ONEFS_FILE_REGEX = re.compile('^(.+)\.json$')


##############################################################################
##  ____                 _ _     _____                           
## |  _ \ ___  ___ _   _| | |_  | ____|_ __  _   _ _ __ ___  ___ 
## | |_) / _ \/ __| | | | | __| |  _| | '_ \| | | | '_ ` _ \/ __|
## |  _ <  __/\__ \ |_| | | |_  | |___| | | | |_| | | | | | \__ \
## |_| \_\___||___/\__,_|_|\__| |_____|_| |_|\__,_|_| |_| |_|___/
##                                                               
##############################################################################

# For simplicity, define only a single enumerated list of result values
# for all possible tables. Originally we had separate lists for "packages"
# and "installs", but easier to define, identify, report error strings if
# we have a single such definition.
#
# DO NOT CHANGE OR DELETE RESULT VALUE ASSIGNMENTS: Only Add!
# WHEN YOU ADD RESULTS, add both defines and dict lookup string.
#
# Note these result values are not monotonically increasing by design.
# E.g., we have a "grouping" of Exception errors starting at 100,
# for easier quick identification when looking at database dumps with
# numeric values.

# IF we do rollback, add RESULT_DELETED for API that "deletes" files

RESULT_UNKNOWN = 0
RESULT_START = 1
RESULT_SUCCESS = 2
RESULT_FAIL = 3

RESULT_MISSING_STAGING_DIR = 11
RESULT_MISSING_SOURCE_FILE = 12
RESULT_MISSING_DEST_DIR = 13
RESULT_MISSING_JSON_TARGET_FILE = 14

RESULT_IFS_NOT_MOUNTED = 21
RESULT_FILE_COPY_ERROR = 22
RESULT_FILE_MISMATCH = 23
RESULT_FILE_SKIPPED = 24

RESULT_EXCEPTION_UNHANDLED = 100
RESULT_EXCEPTION_KEYBOARD = 101
RESULT_EXCEPTION_OS_ERROR = 102
RESULT_EXCEPTION_IO_ERROR = 103

RESULT_PASSING_LIST = [
    RESULT_SUCCESS,
    ]

result_strings = {
    RESULT_UNKNOWN:
    "Unknown/Invalid result state",
    RESULT_START:
    "Start of process",
    RESULT_SUCCESS:
    "Success",
    RESULT_FAIL:
    "Failure",

    RESULT_MISSING_STAGING_DIR:
    "Missing package staging directory",
    RESULT_MISSING_SOURCE_FILE:
    "Missing install source file",
    RESULT_MISSING_DEST_DIR:
    "Missing install destination directory",
    RESULT_MISSING_JSON_TARGET_FILE:
    "Unable to determine install target json filename",

    RESULT_IFS_NOT_MOUNTED:
    "Install target file failed: /ifs is not mounted",
    RESULT_FILE_COPY_ERROR:
    "Install target file failed copy command",
    RESULT_FILE_MISMATCH:
    "Install target file already exists, but does not match package",
    RESULT_FILE_SKIPPED:
    "Install target file already exists, identical, skipped install",

    RESULT_EXCEPTION_UNHANDLED:
    "Unhandled Exception occurred",
    RESULT_EXCEPTION_KEYBOARD:
    "KeyboardInterrupt Exception occurred",
    RESULT_EXCEPTION_OS_ERROR:
    "OSError Exception occurred",
    RESULT_EXCEPTION_IO_ERROR:
    "IOError Exception occurred",
    }

def get_result_string(result):
    try:
        r = int(result)
    except ValueError:
        r = result
    result_string = result_strings.get(r)
    if not result_string:
        result_string = "Unknown or invalid result code: %s" % result
    return result_string


##############################################################################
##  ____                               _      ____          _      
## / ___| _   _ _ __  _ __   ___  _ __| |_   / ___|___   __| | ___ 
## \___ \| | | | '_ \| '_ \ / _ \| '__| __| | |   / _ \ / _` |/ _ \
##  ___) | |_| | |_) | |_) | (_) | |  | |_  | |__| (_) | (_| |  __/
## |____/ \__,_| .__/| .__/ \___/|_|   \__|  \____\___/ \__,_|\___|
##             |_|   |_|                                           
##############################################################################

FuncLine = lambda: inspect.stack()[1][2]
FuncName = lambda: inspect.stack()[1][3]
CallerLine = lambda: inspect.stack()[2][2]
CallerName = lambda: inspect.stack()[2][3]

# get_location_path returns:
# None   : If no matching location found
# String : Full path to matching location directory
# List   : List of full paths to matching location, primary and secondary
def get_location_path(locations, location):
    path = None
    if location in locations:
        path = locations.get(location)
    else:
        for loc in location.split(os.path.sep):
            if loc in locations:
                path = locations = locations.get(loc)
                if isinstance(locations, list):
                    break # Not absolutely necessary, but safer
            else:
                path = None
                break
    return path

def get_ifs_local_paths(paths):
    if not isinstance(paths, list) or not len(paths) == 2:
        error("Invalid ifs/local paths list: %s" % paths)
        return None
    ifs_local_paths = {}
    for path in paths:
        if is_ifs_path(path):
            ifs_local_paths['ifs'] = path
        else:
            ifs_local_paths['local'] = path
    for key in ['ifs', 'local']:
        if not key in ifs_local_paths:
            error("No %s path found in paths list" % key)
            return None
    return ifs_local_paths

def type_str(x):
    try:
        ts = re.match("<type '(.+)'>", str(type(x))).group(1)
    except:
        ts = str(type(x))
    ts = ts.replace('NoneType', 'None')
    return ts

def get_tool_version():
    version = VERSION
    revision = re.sub("\$Revision: (.+) \$", r"r\1", REVISION)
    return ("%s Version %s %s" % (TITLE, version, revision))

def check_tool_version(version):
    valid = True
    if not isinstance(version, int):
        error("Invalid %s version %s" % (TITLE, version))
        valid = False
    elif not (version > 0 and version <= VERSION):
        error("Unsupported %s version %s (tool = %s)" %
              (TITLE, version, VERSION))
        valid = False
    return valid

def system_command_output(args, shell=True, ignore_stderr=False):
    pipe = subprocess.PIPE
    # Due to annoyances with '*' wildcards, and shell=True or False, etc,
    # always pass a string to Popen.
    if isinstance(args, basestring):
        cmd = args
    else:
        cmd = ' '.join(args)
    p = subprocess.Popen(cmd, stdout=pipe, stderr=pipe, shell=shell)
    o, e = p.communicate()
    rc = p.returncode or (not ignore_stderr and e) and 1 or 0
    return (rc, filter(None, e.split('\n')), filter(None, o.split('\n')))

def dump_traceback(msg=None):
    import traceback
    msg and error(msg)
    traceback.print_exc()

def unhandled_exception():
    etype, evalue = sys.exc_info()[:2]
    dump_traceback("Caught Unhandled Exception: %s" % evalue)

def filter_installed_files(filenames, ffiles=INSTALLED_FILES_FILTER_LIST):
    for filter_file in ffiles:
        filenames = remove(filenames, filter_file)
    return filenames

def is_onefs_file(filename):
    match = JSON_ONEFS_FILE_REGEX.match(os.path.basename(filename))
    if match:
        return True
    return False

def is_ifs_path(filepath):
    return str(filepath).startswith('/ifs/')

def not_ifs_path(filepath):
    return not is_ifs_path(filepath)

def ifs_mounted():
    return misc.ifs_mounted()

def transpose(data):
    try:
        tdata = zip(*data)
    except TypeError, e:
        raise TypeError("Unable to transpose data: %s" % e)
    return tdata

def data_column(data, col):
    if not isinstance(col, int):
        raise Exception("Invalid column index %s" % col)
    tdata = transpose(data)
    if not col < len(tdata): # 0-indexed: col = 0..(len-1)
        raise Exception("Invalid column index %s: Data only has %s columns" %
                        (col, len(tdata)))
    return list(tdata[col])

def make_list(data):
    return not isinstance(data, list) and [data] or data

def flatten(data):
    return reduce(lambda a, b: list(a)+list(b), data, [])

def remove(data, element):
    return filter(lambda d: d != element, data)

def remove_duplicates(data):
    typ = type(data)
    return typ(set(data))

def position(data, element):
    return [i for i, d in enumerate(data) if d == element]

def drop(data, pos):
    result = data[:]
    if isinstance(pos, int):
        del result[pos]
    elif isinstance(pos, list):
        for p in sorted(pos, reverse=True):
            del result[p]
    else:
        raise Exception("Unsupported drop position type %s" % type_str(pos))
    return result

def data_width(data):
    return max(map(len, map(str, data)))

def data_widths(data):
    return map(data_width, transpose(data))

def data_dump(data, headers=None, options={}):
    spacing = options.get("spacing", 2)
    alignment = options.get("alignment", "left")
    framed = options.get("framed", False)
    if options.get("titled") == False:
        title = None
    else:
        title = options.get("title", None)
    if headers is not None:
        data = [headers] + data
    widths = data_widths(data)
    formatelement = [alignment=="left" and "%-*s" or "%*s"]
    formatstr = (" " * spacing).join(formatelement * len(data[0]))
    format = lambda d: formatstr % tuple(flatten(transpose([widths, d])))
    if framed:
        frameline = "+-%s-+" % ('-' * len(format([''] * len(widths))))
        framedata = ['-' * w for w in widths]
        formatstr = "| %s |" % formatstr
    else:
        frameline = framedata = None
    if title:
        print(title)
    if frameline:
        print(frameline)
    for d in data:
        print(format(d))
        if d == headers and framedata:
            print(format(framedata))
    if frameline:
        print(frameline)


##############################################################################
##  _               _      _____ _ _           
## | |    ___   ___| | __ |  ___(_) | ___  ___ 
## | |   / _ \ / __| |/ / | |_  | | |/ _ \/ __|
## | |__| (_) | (__|   <  |  _| | | |  __/\__ \
## |_____\___/ \___|_|\_\ |_|   |_|_|\___||___/
##                                             
##############################################################################

_ifs_lockfiles = {}

def ifs_lockfile_get(ifs):
    global _ifs_lockfiles
    return _ifs_lockfiles.get(ifs)

def ifs_lockfile_set(ifs, lockfile):
    global _ifs_lockfiles
    _ifs_lockfiles[ifs] = lockfile

def ifs_lockfile_taken(ifs):
    global _ifs_lockfiles
    return _ifs_lockfiles.get(ifs) and True or False

def ifs_lockfile_open(ifs):
    lockfile = ifs_lockfile_get(ifs)
    if lockfile:
        return lockfile
    err = None
    try:
        if not_ifs_path(ifs):
            err = ("Unable to lock %s: not a valid /ifs path" % ifs)
        elif not ifs_mounted():
            err = ("Unable to lock %s: /ifs is not mounted" % ifs)
        elif not os.path.exists(ifs):
            os.makedirs(ifs)
            if not os.path.exists(ifs):
                err = ("Unable to lock %s: directory does not exist" % ifs)
        if not err and not os.path.isdir(ifs):
            err = ("Unable to lock %s: not a directory" % ifs)
        if not err:
            try:
                filename = os.path.join(ifs, IFS_SYNC_LOCK_FILENAME)
                lockfile = safelockfile(filename, 'w')
                lockfile.write("%s\n" % os.getpid())
                lockfile.flush()
            except IOError, e:
                err = ("Unable to open /ifs sync lock file %s: %s" %
                       (filename, e))
    finally:
        if err:
            error(err)
            if lockfile is not None:
                lockfile.close()
            lockfile = None
        ifs_lockfile_set(ifs, lockfile)
    return lockfile

def ifs_lockfile_close(ifs):
    try:
        lockfile = ifs_lockfile_get(ifs)
        lockfile and lockfile.close()
    finally:
        ifs_lockfile_set(ifs, None)

def ifs_lock_taken():
    return ifs_lockfile_taken(IFS_DSP_DIRECTORY)

def ifs_lock():
    debug("Locking DSP ifs access: %s" % (CallerName()))
    lockfile = ifs_lockfile_open(IFS_DSP_DIRECTORY)
    return lockfile and True or False

def ifs_unlock():
    # If not for debug, this could just be a one-liner:
    # ifs_lockfile_close(IFS_DSP_DIRECTORY)
    if ifs_lockfile_get(IFS_DSP_DIRECTORY):
        debug("Unlocking DSP ifs access: %s..." % (CallerName()))
        ifs_lockfile_close(IFS_DSP_DIRECTORY)

# ifs safe file read and write can be used for both ifs and local files
# If the ifs lockfile is already taken by our process, track this,
# and DO NOT unlock (close) the file -- some other caller has it.

def ifs_safe_file_write(filename, data):
    if not isinstance(filename, basestring):
        error("Invalid filename %s" % filename)
        return 1
    if is_ifs_path(filename) and not ifs_lock_taken():
        lock = ifs_lock()
        if not lock:
            error("Unable to save %s: Cannot get ifs lock" % filename)
            return 1
    else:
        lock = None
    rc = 0
    try:
        path = os.path.dirname(filename)
        if not os.path.exists(path):
            os.makedirs(path)
        if not os.path.exists(path):
            error("Unable to create directory for %s" % filename)
            rc = 1
        else:
            temp = "%s.temp" % filename
            with open(temp, 'w') as f:
                f.writelines(["%s\n" % d.strip('\n') for d in data])
            rcc, err, out = system_command_output(['mv', temp, filename])
            if rcc or err:
                error("Unable to save %s: %s" % (filename, rcc))
                for line in err + out:
                    error(line)
                rc = 1
    finally:
        lock and ifs_unlock()
    return rc

def ifs_safe_file_read(filename):
    if not isinstance(filename, basestring):
        error("Invalid filename %s" % filename)
        return None
    if not os.path.exists(filename) or not os.path.isfile(filename):
        error("Unable to read %s: File does not exist" % filename)
        return None
    if is_ifs_path(filename) and not ifs_lock_taken():
        lock = ifs_lock()
        if not lock:
            error("Unable to save %s: Cannot get ifs lock" % filename)
            return None
    else:
        lock = None
    data = None
    try:
        with open(filename, 'r') as f:
            data = f.readlines()
    finally:
        lock and ifs_unlock()
    return data


##############################################################################
##  _                                   ____ _               
## | |    ___   __ _  __ _  ___ _ __   / ___| | __ _ ___ ___ 
## | |   / _ \ / _` |/ _` |/ _ \ '__| | |   | |/ _` / __/ __|
## | |__| (_) | (_| | (_| |  __/ |    | |___| | (_| \__ \__ \
## |_____\___/ \__, |\__, |\___|_|     \____|_|\__,_|___/___/
##             |___/ |___/                                   
##############################################################################

class Logger(object):

    def __init__(self, progname, logfile, level, format, datefmt, verbose):
        self._error_count = 0
        self._logger = None
        self._level = level
        self._verbose = verbose
        try:
            self._logger = logging.getLogger(progname)
            self._logger.setLevel(level)
            logformat = logging.Formatter(format, datefmt=datefmt)
            loghandle = logging.FileHandler(logfile)
            loghandle.setFormatter(logformat)
            self._logger.addHandler(loghandle)
        except IOError:
            print("IOError during logger init: %s, %s" % (progname, logfile))
            self._logger = None

    def set_level(self, level):
        self._level = level
        if self._logger:
            self._logger.setLevel(level)

    def set_verbose(self, verbose):
        self._verbose = verbose

    def log(self, msg, level=logging.INFO, console=True, logfile=True):
        if self._logger and logfile:
            self._logger.log(level, "%s: %s" % (PROGNAME, msg))
        if self._verbose and console:
            # Ok this is from non-obvious error reporting choices, driven by
            # specific requests regarding applying and reporting data value
            # overrides with drive config files.  Normally, we would decide
            # that an "err" (bool var) == (level >= logging.ERROR).  These
            # "err's" are reported to sys.stderr, rather than sys.stdout.
            # However, a request was made that if there are "unable to apply
            # override" conflicts in InstalledFiles.apply_data_overrides(),
            # we should log 1. Report an 'error' to the console via stderr,
            # 2. NOT report a non-zero error indicator return code, 3. Still
            # report the modified drive config file data to the console for
            # the consumer to use. At first, warn() seemed a solution to this
            # requirement: It reports a "special" error to console, it reports
            # a "special" error to log file, and it does NOT return a non-zero
            # error code. However, err=(level >= logging.ERROR) will report
            # warn() to stdout, not stderr.  Therefore, we have the following:
            #      level    console  errorcode
            #      -------- -------- -------- 
            #      >= ERROR stderr   non-0
            #      == WARN  stderr   0
            #      <  WARN  stdout   0
            report_error  = level >= logging.ERROR
            console_error = level >= logging.WARN
            stdconsole = console_error and sys.stderr or sys.stdout
            err = level >= logging.ERROR
            if report_error:
                self._error_count += 1
            tag = {
                logging.CRITICAL: 'CRITICAL: ',
                logging.ERROR: 'ERROR: ',
                logging.WARN: 'WARN: ',
                logging.INFO: '',
                logging.DEBUG: 'DEBUG: ',
                }.get(level, '')
            print >> stdconsole, tag+msg

    def critical(self, msg):
        self.log(msg, level=logging.CRITICAL)

    def error(self, msg):
        self.log(msg, level=logging.ERROR)

    def warn(self, msg):
        self.log(msg, level=logging.WARN)

    def debug(self, msg):
        if self._level <= logging.DEBUG:
            self.log(msg, level=logging.DEBUG)

    def error_count(self):
        return self._error_count

    def close(self):
        logging.shutdown()

    def __del__(self):
        self.close()

_logger = None

def get_logger(progname=PROGNAME, logfile=LOG_FILE,
               level=LOG_LEVEL, format=LOG_FORMAT,
               datefmt=LOG_FORMAT_DATE, verbose=VERBOSE):
    global _logger
    if not _logger:
        _logger = Logger(progname, logfile, level, format, datefmt, verbose)
    return _logger

def log(msg, level=logging.INFO, console=True, logfile=True):
    logger = get_logger()
    logger and logger.log(msg, level, console=console, logfile=logfile)

def critical_error(msg):
    logger = get_logger()
    logger and logger.critical(msg)

def error(msg):
    logger = get_logger()
    logger and logger.error(msg)

def warn(msg):
    logger = get_logger()
    logger and logger.warn(msg)

def debug(msg):
    logger = get_logger()
    logger and logger.debug(msg)

##############################################################################
##  ____        _        _                       ____ _               
## |  _ \  __ _| |_ __ _| |__   __ _ ___  ___   / ___| | __ _ ___ ___ 
## | | | |/ _` | __/ _` | '_ \ / _` / __|/ _ \ | |   | |/ _` / __/ __|
## | |_| | (_| | || (_| | |_) | (_| \__ \  __/ | |___| | (_| \__ \__ \
## |____/ \__,_|\__\__,_|_.__/ \__,_|___/\___|  \____|_|\__,_|___/___/
## 
##############################################################################

# Notes on single- and double-quotes in Database code:
#    'keyword'  A keyword in single quotes is a string literal.
#    "keyword"  A keyword in double quotes is an identifier.
# SQLite uses the SQL standard for quoting string literals: single-quotes.
# SQLite uses double-quotes around identifiers (columns or table names)
# that contain special characters or which are keywords.
# For sanity and readability, please keep to a simple convention:
# ** Use single-quotes for SQL commands and quoting
# ** Use double-quotes for code (format strings, function arguments, etc)

def db_time():
    return int(time.time())

class DatabaseError(Exception):
    pass

class Database(object):

    DB_INDEX = ("id", "INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL")

    DB_COLUMN_HEADERS = ('id', 'name', 'type', 'null', 'default', 'auto')

    # If database does not exist, open() will raise DatabaseError exception.
    # newfile = True : Always create a new database file, overwriting existing
    # newfile = Auto : If database file does not exist, create a new one
    # newfile = False: Use existing database file
    def __init__(self, database_file, newfile='Auto'):
        database_file = str(database_file)
        self._conn = self._cursor = None
        self._file = database_file
        self.debug(database_file)
        if database_file == "None":
            raise DatabaseError("Database is not supported")
        if str(newfile).lower()[:4] == "auto":
            newfile = not os.path.exists(database_file)
        if newfile is True:
            parent_dir = os.path.dirname(database_file)
            if not os.path.exists(parent_dir):
                try:
                    os.makedirs(parent_dir)
                except:
                    raise DatabaseError("Unable to create database dir %s" %
                                        parent_dir)
            try:
                with open(database_file, "w") as f:
                    f.write("")
            except:
                raise DatabaseError("Unable to create database %s" %
                                    database_file)
        self._conn = self._cursor = None
        if os.path.exists(database_file):
            self.conn()

    def quote_literal(self, keyword):
        """Returns single-quoted string (for SQLite string literals)"""
        return "'%s'" % str(keyword).strip("'")

    def debug(self, msg):
        """Logs debug message (if debug is enabled)"""
        # Uses global debug function
        debug("DB:%s: %s" % (CallerName(), msg))

    def DEV_DEBUG(self, msg, level=1):
        if False and level <= 1:
            debug("DB:%s: %s" % (CallerName(), msg))

    def conn(self):
        """Returns connection to database"""
        # XXX not a valid db_file, db_file is a dir, etc - how to handle
        # XXX What about timeouts. What about multiple connections (?!?)
        if not self._conn:
            self._conn = sqlite3.connect(self._file)
        return self._conn

    def destroy(self):
        """Destroys (closes) connection to database"""
        try:
            self._conn and self._conn.close()
        finally:
            self._conn = None

    def open(self):
        """Returns cursor to database"""
        if not self._conn:
            raise DatabaseError("No database connection to %s" % self._file)
        if not self._cursor:
            self._cursor = self._conn.cursor()
        return self._cursor

    def close(self):
        """Closes cursor to database"""
        try:
            self._cursor and self._cursor.close()
        finally:
            self._cursor = None

    def validate(self, conn=True, cursor=True):
        """Verify database connectivity"""
        if conn and not self._conn:
            raise DatabaseError("No database connection to %s" % self._file)
        if cursor and not self._cursor:
            raise DatabaseError("No database cursor to %s" % self._file)
        return True

    def execute(self, sql, parameters=None):
        """Executes database command"""
        if not self.validate():
            return None
        if parameters is None:
            self.DEV_DEBUG(sql)
            return self._cursor.execute(sql)
        else:
            self.DEV_DEBUG((sql, parameters))
            return self._cursor.execute(sql, parameters)

    def executemany(self, sql, parameters):
        """Executes multiple database commands using sequence parameters"""
        if not self.validate():
            return None
        if not parameters: # Actually needs to be seq/mapping/iterator
            raise DatabaseError("Invalid parameters for executemany")
        else:
            self.debug((sql, parameters))
            return self._cursor.executemany(sql, parameters)

    def commit(self):
        """Commits data to database"""
        self._conn and self._conn.commit()

    def fetchone(self):
        """Returns next result from last query, or None if no data avail"""
        if self.validate():
            return None
        return self._cursor.fetchone()

    def fetchmany(self):
        """Returns set of results from last query as a list"""
        if not self.validate():
            return None
        return self._cursor.fetchmany()

    def fetchall(self):
        """Returns all results from last query as a list"""
        if not self.validate():
            return None
        return self._cursor.fetchall()

    def select(self, results, source, where=None,
               groupby=None, having=None,
               orderby=None, limit=None, offset=None,
               parameters=None):
        """Executes database SELECT command"""
        if isinstance(results, list):
            results = ", ".join(results)
        where = where and "WHERE %s" % where
        groupby = groupby and "GROUP BY %s" % groupby
        having = having and "HAVING %s" % having
        orderby = orderby and "ORDER BY %s" % orderby
        limit = limit and "LIMIT %s" % limit
        offset = offset and "OFFSET %s" % offset
        options = [where, groupby, having, orderby, limit, offset]
        options = " ".join(filter(None, options))
        sql = ("SELECT %s FROM %s %s" % (results, source, options))
        return self.execute(sql, parameters)

    def tables(self):
        """Returns list of database tables"""
        name = self.quote_literal("sqlite_master")
        where = "type=%s" % self.quote_literal("table")
        self.select("name", name, where=where, orderby="name")
        results = self.fetchall() or []
        return remove([result[0] for result in results], "sqlite_sequence")

    def columns(self, table):
        """Returns list of database table columns"""
        sql = ("PRAGMA table_info(%s)" % table)
        self.execute(sql)
        results = self.fetchall() or []
        return results 

    def create_table(self, table, definition, options=[]):
        """Creates given database table using definition"""
        self.debug(table)
        colf = lambda d: " ".join([self.quote_literal(d[0])] + list(d[1:]))
        spec = ", ".join(map(colf, definition))
        optf = lambda o: isinstance(o, basestring) and o or "=".join(o)
        opts = ", ".join(map(optf, options))
        sql = ("CREATE TABLE IF NOT EXISTS %s (%s) %s" % (table, spec, opts))
        return self.execute(sql)

    def insert_row(self, table, values, autoindex=True):
        """Insert new values into given table"""
        vals = (autoindex is True and ["NULL"] or []) + ["?"] * len(values)
        vals = ", ".join(vals)
        table = self.quote_literal(table)
        sql = ("INSERT INTO %s VALUES (%s)" % (table, vals))
        x = self.execute(sql, values)
        x and self.DEV_DEBUG("rowcount=%s, rowid=%s" %
                             (x.rowcount, x.lastrowid), 3)
        return x

    def update_row(self, table, id, values):
        """Update table data for given id with new values"""
        self.debug(table)
        data = ["%s=%s" % (key, val) for key, val in values.iteritems()]
        data = ", ".join(data)
        table = self.quote_literal(table)
        where = "%s=%s" % ("id", id)
        sql = ("UPDATE %s SET %s WHERE %s" % (table, data, where))
        x = self.execute(sql)
        x and self.DEV_DEBUG("rowcount=%s, rowid=%s" %
                             (x.rowcount, x.lastrowid), 3)
        return x

    def data_dump(self, data, headers, title=None, options={}):
        # Currently the only user of return result list is UnitTest
        # Get result (for UnitTest right now) before data mods
        result = [dict(transpose([headers, d])) for d in data]
        tindex = position(headers, "time")
        tindex = tindex and tindex[0] or None
        if tindex is not None:
            displaytime = {
                "raw":0, "false":0, # Only raw numeric
                "format":1,         # Only formatted
                "both":2, "true":2, # Both raw and formatted
                }.get(str(options.get("time", True)).lower(), 0)
            timefmt = lambda t: time.ctime(t)
            def timedata(d):
                td = list(d)
                t = td[tindex]
                if displaytime == 1:
                    t = timefmt(t)
                elif displaytime == 2:
                    t = "%s=%s" % (t, timefmt(t))
                else: # default = raw numeric
                    pass
                td[tindex] = t
                return tuple(td)
            data = map(timedata, data)
        if options.get("console", True):
            options["title"] = title
            data_dump(data, headers, options)
        return result

    def dump_table_rows(self, table, rows=None, options={}):
        """Print and return row data for given table"""
        title = "Row data for table %s:" % table
        columns = [col[1] for col in self.columns(table)]
        if rows is None:
            self.select("*", table)
            rows, rowsread = [], self.fetchmany()
            while rowsread:
                rows.extend(rowsread)
                rowsread = self.fetchmany()
        drop_columns = options.get("drop_columns", {}).get(table, [])
        drop_pos = flatten([position(columns, c) for c in drop_columns])
        if drop_pos:
            columns = drop(columns, drop_pos)
            rows = [tuple(drop(list(row), drop_pos)) for row in rows]
        return self.data_dump(rows, columns, title, options)

    def dump_table_columns(self, table, options={}):
        """Print and return column data (definition) for given table"""
        title = "Table %s:" % table
        headers = self.DB_COLUMN_HEADERS
        columns = self.columns(table)
        return self.data_dump(columns, headers, title, options)

    def dump_tables(self, options={}):
        """Print and return all database table definitions"""
        result = {}
        print("Database Table Definitions:")
        tables = sorted(self.tables())
        for table in tables:
            result[table] = self.dump_table_columns(table, options)
        return result

    def __del__(self):
        self.commit()
        self.close()
        self.destroy()


class DSPDatabase(Database):

    db_versions = {
        1: {"packages":
            [ Database.DB_INDEX,
              ("version", "INTEGER"),
              ("package", "TEXT"),
              ("result", "INTEGER"),
              ("count", "INTEGER"),
              ("time", "INTEGER"),
              ],
            "locations":
            [ Database.DB_INDEX,
              ("version", "INTEGER"),
              ("identifier", "TEXT"),
              ("location", "TEXT"),
              ],
            "installs":
            [ Database.DB_INDEX,
              ("pkg", "INTEGER"),
              ("loc", "INTEGER"),
              ("filename", "TEXT"),
              ("version", "INTEGER"),
              ("result", "INTEGER"),
              ("count", "INTEGER"),
              ("time", "INTEGER"),
              ]
            },
        }

    def __init__(self, database_file, version, newfile='Auto'):
        if not check_tool_version(version):
            raise DatabaseError("Invalid version %s" % version)
        Database.__init__(self, database_file, newfile)
        self._version = version
        self.create_tables()

    def table_definitions(self, version=None):
        """Returns table definitions for current version"""
        if version is None:
            version = self._version
        return self.db_versions.get(version, None)

    def table_definition(self, table, version=None):
        """Returns table definition for given table, version"""
        if version is None:
            version = self._version
        return self.db_versions.get(version, {}).get(table, None)

    def table_values(self, table, parameters, version=None):
        """Returns table values for given table and parameters"""
        definition = self.table_definition(table, version=version)
        if not definition:
            raise DatabaseError("No known table definition for %s" % table)
        required = remove(definition, self.DB_INDEX)
        if not required:
            raise DatabaseError("Invalid table definition for %s" % table)
        required = data_column(required, 0)
        params = parameters.copy()
        if not "version" in params:
            params["version"] = version or self._version
        missing = [[reqd not in params, reqd] for reqd in required]
        missing = filter(lambda m: m[0], missing)
        if missing:
            missing = data_column(missing, 1)
            raise DatabaseError("Missing required %s parameters: %s" %
                                (table, ", ".join(missing)))
        return [params[key] for key in required]

    def table_id_exists(self, table, id):
        """Returns true if row id exists in table"""
        x = self.select("id", table, where="id=%s" % id)
        exists = x and self.fetchall() # len(rows) > 0
        return exists

    def table_row(self, table, id):
        """Returns table row data as a dict for given id"""
        x = self.select("*", table, where="id=%s" % id)
        rows = x and self.fetchall()
        if not rows:
            raise DatabaseError("Table %s id %s does not exist" % (table, id))
        if not len(rows) == 1:
            raise DatabaseError("Table %s id %s has %d rows???" % (table, id,
                                                                   len(rows)))
        definition = self.table_definition(table)
        row = dict(transpose([data_column(definition, 0), rows[0]]))
        self.DEV_DEBUG((("table", table), ("id", id), ("row", row)), 3)
        return row

    def table_row_id(self, table, parameters, version=None,
                     search=None, autoindex=True):
        """Returns table row id matching given parameters, else new row id"""
        rowid = None
        # Determine if this defined row (values) exists
        # If exists, return the unique rowid (or if multiple, the last)
        # If not exists, insert a new row and return new rowid
        values = self.table_values(table, parameters, version=version)
        self.DEV_DEBUG((table, parameters), 5)
        self.DEV_DEBUG((table, values), 5)
        search = ["%s=%s" % (s, self.quote_literal(parameters[s]))
                  for s in remove(search or parameters.keys(), "id")]
        x = self.select("id", table, where=" AND ".join(search))
        if x:
            rows = self.fetchall()
            self.DEV_DEBUG(("rows", rows), 5)
            if rows:
                rowid = rows[-1][0]
                self.DEV_DEBUG(("rowid", rowid), 5)
        if rowid is None:
            self.DEV_DEBUG("inserting data / getting new rowid", 5)
            x = self.insert_row(table, values, autoindex)
            if x:
                rowid = x.lastrowid
                self.DEV_DEBUG(("new rowid", rowid), 5)
        return rowid

    def create_tables(self, version=None):
        """Create all database tables based on current version"""
        definitions = self.table_definitions(version=version)
        if not definitions:
            raise DatabaseError("No known table definitions")
        try:
            self.open()
            for table, definition in definitions.iteritems():
                self.create_table(table, definition)
        finally:
            self.close()

    def package_id(self, parameters):
        """Returns matching or new packages row id for given parameters"""
        search = ["package"]
        return self.table_row_id("packages", parameters, search=search)

    def location_identifier(self, ident):
        """Returns locations table identifier string"""
        self.DEV_DEBUG("identifier=%s" % ident, 3)
        if not ident:
            raise DatabaseError("Invalid locations identifier: %s" % ident)
        elif isinstance(ident, list):
            return os.path.join(*ident)
        elif isinstance(ident, basestring):
            return ident
        raise DatabaseError("Invalid locations identifier: %s" % ident)

    def location_id(self, parameters):
        """Returns matching or new locations row id for given parameters"""
        return self.table_row_id("locations", parameters)

    def install_id(self, parameters):
        """Returns matching or new installs row id for given parameters"""
        search = ['pkg', 'loc', 'filename', 'version']
        return self.table_row_id("installs", parameters, search=search)

    def update_package(self, id, result=None, timestamp=None):
        """Updates packages table data for given row id"""
        table = "packages"
        # Get current count, increment on update
        row = self.table_row(table, id)
        count = row.get("count", 0) + 1
        values = {"count": count}
        if result is not None:
            values.update({"result": result})
        if timestamp is not None:
            values.update({"time": timestamp})
        if values:
            return self.update_row(table, id, values)

    def update_install(self, id, result=None, timestamp=None, version=None):
        """Updates installs table data for given row id"""
        table = "installs"
        # Get current count, increment on update
        row = self.table_row(table, id)
        count = row.get("count", 0) + 1
        values = {"count": count}
        if result is not None:
            values.update({"result": result})
        if timestamp is not None:
            values.update({"time": timestamp})
        if version is not None:
            values.update({"version": version})
        if values:
            return self.update_row(table, id, values)


class UnitTestDatabase(DSPDatabase):

    def __init__(self, database_file, version, newfile=True):
        DSPDatabase.__init__(self, database_file, version, newfile)

    def UnitTableTestPositive(self, table, tests,
                                   select_column, select_value, select_expd,
                                   verbose=False):
        rc = 0
        for test in tests:
            id = self.table_row_id(table, test)
            if verbose:
                print("%s: %s" % (id, test))
        self.commit()
        x = self.select("*", table, where="%s=?" % select_column,
                        parameters=(select_value,))
        rows = self.fetchall()
        self.dump_table_rows(table, rows, framed=True)
        rowids = data_column(rows, 0)
        if rowids != select_expd:
            print("FAILURE: %s: Expected row ids %s, actual %s" %
                  (table, select_expd, rowids))
            rc = 1
        else:
            print("SUCCESS: %s: All row insertion / selection as expected" %
                  (table))
        return rc

    def UnitTableTestNegative(self, table, tests, verbose=False):
        fails = []
        width = max([len(str(test[1])) for test in tests])
        # XXX TODO replace width calc above with appropriate, after verify
        for test, desc in tests:
            id = exc = None
            try:
                id = self.table_row_id(table, test)
            except Exception as exc:
                pass
            if verbose:
                print("%-*s: id=%-4s: exc=%s" % (width, desc, id, exc))
            if id is not None or exc is None:
                fails.append("FAILURE: %s: %s" % (table, test))
            elif exc and "Missing:" in desc:
                mexpd = re.sub("^.*Missing.*:[ ]*", "", desc)
                mexpd = sorted([s.strip() for s in mexpd.split(',')])
                missing = re.sub("Missing required.*parameters:[ ]*", "", str(exc))
                missing = sorted([s.strip() for s in missing.split(',')])
                if mexpd != missing:
                    fails.append("FAILURE: %s: Missing expected %s, actual %s" %
                                 (table, mexpd, missing))
        if fails:
            rc = 1
            for fail in fails:
                print(fail)
        else:
            print("SUCCESS: %s: All negative test cases failed as expected" %
                  (table))
        return fails and 1 or 0

    def UnitTest(self):
        rc = 0
        ver = self._version
        self.open()
        tables = self.dump_tables(framed=True)
        expd_tables = self.table_definitions(version=ver)
        expd_tablenames = sorted(expd_tables.keys())
        for table in expd_tablenames:
            expd_columns = data_column(expd_tables[table], 0)
            actual = tables.get(table)
            if not actual:
                rc = 1
                print("FAILED: %s: Missing table" % table)
            else:
                columns = [col.get('name') for col in actual]
                if expd_columns != columns:
                    rc = 1
                    print("FAILED: %s: Mismatch columns: %s vs %s" %
                          (expd_columns, columns))
        table = "packages"
        print("Testing table %s" % table)
        tpver2, tpver3 = "2.3", "3.0"
        # NOTE: currently there is no protection validity check on "version"
        tests = [
            {"version":ver, "package":tpver2, "result":3, "time":12345678L},
            {"version":ver, "package":tpver2, "result":1, "time":333444555},
            {"version":ver, "package":tpver2, "result":1, "time":333444555},
            {"version":ver, "package":tpver3, "result":0, "time":888877777},
            {"version":ver, "package":tpver2, "result":1, "time":777788888},
            {"version":ver, "package":tpver3, "result":0, "time":888877777, "extra":23},
            # If version is not specified, code will use current version
            {               "package":tpver2, "result":1, "time":777788888},
            ]
        col, val, expd = "package", tpver2, [1, 2, 3, 5, 7]
        if self.UnitTableTestPositive(table, tests, col, val, expd):
            rc = 1
        tests = [
            ({"version":ver, "package":tpver2, "result":1                  },
             "Missing: time"),
            ({"version":ver, "package":tpver2,             "time":222222222},
             "Missing: result"),
            ({"version":ver,                   "result":3, "time":333333333},
             "Missing: package"),
            # If version is not specified, code will use current version
            #({"version":-1, "package":tpver3, "result":5, "time":555555555},
            # "Invalid: version"),
            # => Currently there is no validity check on "version"
            ({"version":ver,                                               },
             "Missing: package, result, time"),
            ]
        if self.UnitTableTestNegative(table, tests):
            rc = 1
        table = "locations"
        print("Testing table %s" % table)
        # XXX XXX FIXME CHANGES
        path1, path2 = LOCATIONS['config']['drives'], LOCATIONS['firmware']
        iden1, iden2 = "config/drives", "firmware"
        tests = [
            {"version":ver, "identifier":iden1, "location":path1+"modelfoo.json"},
            {"version":ver, "identifier":iden1, "location":path1+"modelbar.json"},
            {"version":ver, "identifier":iden2, "location":path2+"modelfoo/bin.abc"},
            {"version":ver, "identifier":iden2, "location":path2+"modelbar/path/bin.def"},
            {"version":ver, "identifier":iden2, "location":path2+"modelbar/path/bin.def", "extra":"foo"},
            {"version":ver, "identifier":iden1, "location":path1+"modelbar.json.1"},
            # If version is not specified, code will use current version
            {               "identifier":iden1, "location":path1+"modelbar.json.1"},
            ]
        col, val, expd = "identifier", iden1, [1, 2, 6, 7]
        if self.UnitTableTestPositive(table, tests, col, val, expd):
            rc = 1
        tests = [
             ({"version":ver, "identifier":iden1, },
              "Missing: location"),
             ({"version":ver,                     "location":path1+"modelfoo.json"},
              "Missing: identifier"),
             # If version is not specified, code will use current version
             ]
        if self.UnitTableTestNegative(table, tests):
            rc = 1
        table = "installs"
        print("Testing table %s" % table)
        tests = [
            {"pkg":1, "loc":1, "filename":"fname1", "version":1, "result":1, "time":1111},
            {"pkg":1, "loc":2, "filename":"fname2", "version":2, "result":0, "time":2222},
            {"pkg":1, "loc":3, "filename":"fname3", "version":3, "result":1, "time":3333},
            {"pkg":1, "loc":4, "filename":"fname4", "version":1, "result":2, "time":4444},
            {"pkg":1, "loc":5, "filename":"fname5", "version":2, "result":3, "time":5555},
            {"pkg":1, "loc":6, "filename":"fname6", "version":3, "result":0, "time":6666},
            {"pkg":1, "loc":7, "filename":"fname7", "version":1, "result":1, "time":7777, "extra":"foo"},
            ]
        col, val, expd = "result", 1, [1, 3, 7]
        if self.UnitTableTestPositive(table, tests, col, val, expd):
            rc = 1
        tests = [
            ({"pkg":1, "loc":1, "filename":"fname1", "version":1, "result":1,            },
             "Missing: time"),
            ({"pkg":1, "loc":1, "filename":"fname1", "version":1,             "time":1111},
             "Missing: result"),
            ({"pkg":1, "loc":1, "filename":"fname1",               "result":1, "time":1111},
             "Missing: version"),
            ({"pkg":1, "loc":1,                      "version":1, "result":1, "time":1111},
             "Missing: filename"),
            ({"pkg":1,               "filename":"fname1", "version":1, "result":1, "time":1111},
             "Missing: loc"),
            ({             "location":1, "filename":"fname1", "version":1, "result":1, "time":1111},
             "Missing: pkg"),
            ]
        if self.UnitTableTestNegative(table, tests):
            rc = 1
        self.close()
        return rc

def get_database(version, database_file, newfile=False, options={}):
    database_file = str(database_file)
    if not check_tool_version(version):
        # Errors reported in code
        return None
    if database_file == "None":
        error("Database is not supported")
        return None
    if not newfile and not os.path.exists(database_file):
        error("Database %s does not exist" % database_file)
        return None
    return DSPDatabase(database_file, version, newfile)


##############################################################################
##      _ ____   ___  _   _   ____                               _   
##     | / ___| / _ \| \ | | / ___| _   _ _ __  _ __   ___  _ __| |_ 
##  _  | \___ \| | | |  \| | \___ \| | | | '_ \| '_ \ / _ \| '__| __|
## | |_| |___) | |_| | |\  |  ___) | |_| | |_) | |_) | (_) | |  | |_ 
##  \___/|____/ \___/|_| \_| |____/ \__,_| .__/| .__/ \___/|_|   \__|
##                                       |_|   |_|                   
##############################################################################

def json_file(filename):
    match = JSON_FILE_REGEX.match(os.path.basename(filename))
    if not match:
        return False
    model, ver = match.groups()
    if ver is not None:
        ver = int(ver.strip('.'))
    return (model, ver)

def json_onefs_file(filename):
    match = JSON_ONEFS_FILE_REGEX.match(os.path.basename(filename))
    if not match:
        return False
    return True

def json_file_data(filename):
    name = "json file %s" % filename
    if os.path.exists(filename):
        data = ifs_safe_file_read(filename)
        try:
            if data:
                jobj = json.loads("\n".join(data))
            else:
                warn("%s is not readable" % name)
                jobj = None
        except ValueError: # No JSON object could be decoded
            # Use warn instead of error - many functions get json_file_ver
            # for files before processing, such as display, matching, selected
            warn("%s does not contain JSON data" % name)
            jobj = None
    else:
        error("%s does not exist" % name)
        jobj = None
    return jobj

def json_file_ver(filename):
    ver = None
    jobj = json_file_data(filename)
    if jobj:
        ver = jobj.get('version')
        if ver is None:
            error("%s does not contain a version" % name)
        else:
            try:
                ver = int(ver)
            except ValueError:
                error("%s contains invalid version %s" % (name, ver))
                ver = None
    return ver

def json_target_filename(filename):
    if not json_file(filename):
        return None
    version = json_file_ver(filename)
    if version is None:
        return None
    model, filename_ver = json_file(filename)
    if filename_ver is None or filename_ver == version:
        return "%s.json.%s" % (model, version)
    error("json file %s has filename version %s, "
          "but file contents version is %s" %
          (filename, filename_ver, version))
    return None

def json_target_filepath(source, target):
    filename = json_target_filename(source)
    if filename:
        filepath = os.path.join(os.path.dirname(target), filename)
    else:
        error("Unable to determine json target filename for %s" % source)
        filepath = None
    return filepath

# Loosely based on email code from Anthony Beltran
def json_file_hash(filename):
    exclude_key = 'hash'
    hash = jobj = data = None
    if os.path.exists(filename):
        data = ifs_safe_file_read(filename)
    else:
        error("%s does not exist" % name)
        data = None
    if data:
        try:
            jobj = json.loads("\n".join(data))
        except ValueError: # No JSON object could be decoded
            error("%s does not contain JSON data" % name)
            jobj = None
    if jobj:
        if exclude_key in jobj:
            del jobj[exclude_key]
        hash = hashlib.md5(json.dumps(jobj, sort_keys=True)).hexdigest()
    return hash


##############################################################################
##  ____                     ___  __     
## / ___| _   _ _ __   ___  |_ _|/ _|___ 
## \___ \| | | | '_ \ / __|  | || |_/ __|
##  ___) | |_| | | | | (__   | ||  _\__ \
## |____/ \__, |_| |_|\___| |___|_| |___/
##        |___/                          
##############################################################################

# Current approach, sync_ifs_local() calls sync_ifs_local_dirs() for specifc
# dirs, we will be locking and unlocking lockfile for each dir, not per node.
# Don't push locking all the way down (to sync_ifs_source_dest) -- we'd like
# to keep this "directory-level sync" atomic, per node -- don't want any churn
# from other nodes on (source, dest) directory listings mid-os.walk(source).

def sync_ifs_source_dest(source, dest, options={}):
    rc = 0
    if options.get(LOCAL_FLAG):
        return rc
    is_ifs = is_ifs_path(dest)
    for dirpath, dirnames, filenames in os.walk(source):
        filenames = filter_installed_files(filenames)
        # If syncing FROM local TO ifs, DO NOT copy OneFS files
        if is_ifs:
            filenames = filter(lambda f: not is_onefs_file(f), filenames)
        for filename in filenames:
            sfile = os.path.join(dirpath, filename)
            dfile = sfile.replace(source, dest)
            exists = os.path.exists(dfile)
            if exists and not options.get('overwrite'):
                continue
            try:
                tofrom = is_ifs and "to" or "from"
                cpfile = is_ifs and dfile or sfile
                debug("Sync copy %s IFS: %s" % (tofrom, cpfile))
                cmd = ["cp", "-fp", quote(sfile), quote(dfile)]
                rcc, err, out = system_command_output(cmd)
                if rcc or err:
                    error("Unable to sync copy %s IFS: %s" % (tofrom, cpfile))
                    debug((cmd, rcc, err, out))
                    rc = 1
            except:
                unhandled_exception()
                rc = 1
    return rc

def sync_ifs_local_dirs(ifs, local, options={}):
    rc = 0
    if options.get(LOCAL_FLAG):
        return rc
    if not ifs_mounted():
        msg = "Unable to sync ifs and local dirs: ifs is not mounted"
        if WARN_IF_IFS_UNMOUNTED:
            warn(msg)
        else:
            error(msg)
            rc = 1
        return rc
    lock = ifs_lock()
    if not lock:
        error("Unable to sync %s: Cannot get ifs lock" % ifs)
        return 1
    if not os.path.exists(ifs):
        os.makedirs(ifs)
    if not os.path.exists(local):
        os.makedirs(local)
    try:
        if sync_ifs_source_dest(ifs, local, options):
            rc = 1
        if sync_ifs_source_dest(local, ifs, options):
            rc = 1
    except IOError, e:
        error("Unable to sync %s: %s" % (ifs, e))
        rc = 1
    finally:
        ifs_unlock()
    return rc

def sync_ifs_local_dirs_list(dirs, options={}):
    rc = 0
    if options.get(LOCAL_FLAG):
        return rc
    paths = get_ifs_local_paths(dirs)
    if not paths:
        error("Invalid sync ifs dirs list: %s" % dirs)
        rc = 1
    elif sync_ifs_local_dirs(paths.get('ifs'), paths.get('local'), options):
        rc = 1
    return rc

def sync_ifs_local(locations, options={}, locn=[]):
    rc = 0
    if options.get(LOCAL_FLAG):
        return rc
    location_keys = sorted(locations.keys())
    for location in location_keys:
        nextlocn = locn + [location]
        dest = get_location_path(locations, location)
        if dest is None:
            continue
        if isinstance(dest, dict):
            if sync_ifs_local(dest, options, nextlocn):
                rc = 1
        elif isinstance(dest, basestring):
            pass # Do nothing, no sync
        elif isinstance(dest, list):
            if sync_ifs_local_dirs_list(dest, options):
                rc = 1
        else:
            error("Invalid location %s" % dest)
            rc = 1
    return rc


##############################################################################
##  _____ _ _         ___                       _   _                 
## |  ___(_) | ___   / _ \ _ __   ___ _ __ __ _| |_(_) ___  _ __  ___ 
## | |_  | | |/ _ \ | | | | '_ \ / _ \ '__/ _` | __| |/ _ \| '_ \/ __|
## |  _| | | |  __/ | |_| | |_) |  __/ | | (_| | |_| | (_) | | | \__ \
## |_|   |_|_|\___|  \___/| .__/ \___|_|  \__,_|\__|_|\___/|_| |_|___/
##                        |_|                                         
##############################################################################

def quote(qstr):
    if '"' in qstr:
        q = "'%s'" % qstr.strip("'")
    else:
        q = '"%s"' % qstr.strip('"')
    return q

def file_compare_match(source, target):
    match = False
    if json_file(source) or json_file(target):
        hash_source = json_file_hash(source)
        hash_target = json_file_hash(target)
        match = hash_source == hash_target
    else:
        cmd = ["cmp", "-s", quote(source), quote(target)]
        rc, err, out = system_command_output(cmd)
        match = rc == 0
    return match

# file version defaults to 0 for NA/unknown
def file_ver(filepath):
    if not os.path.exists(filepath) or not os.path.isfile(filepath):
        return None
    if not json_file(filepath):
        return 0
    ver = json_file_ver(filepath)
    # ver could be None on error
    return ver

def copy_file(database, pkg_id, loc_id, source, destination, filepath,
              create_dir=True, overwrite=False):
    file_id = None
    result = RESULT_UNKNOWN
    target = filepath.replace(source, destination)
    filename = filepath.replace(source, "").lstrip('/')
    version = file_ver(filepath)
    action, compare = "Install", ""
    try:
        # Always get db access id, even on error, for error updates
        if database:
            file_id = database.install_id({
                "pkg": pkg_id, "loc": loc_id,
                "filename": filename, "version": version,
                "result": RESULT_START, "count": 0, "time": db_time(),
                })
        if result is RESULT_UNKNOWN:
            if not os.path.exists(str(filepath)):
                result = RESULT_MISSING_SOURCE_FILE
        if result is RESULT_UNKNOWN:
            # Verify and Create target path if necessary
            path = os.path.dirname(str(target))
            if is_ifs_path(path) and not ifs_mounted():
                result = RESULT_IFS_NOT_MOUNTED
        if result is RESULT_UNKNOWN:
            if not os.path.exists(path) and create_dir:
                os.makedirs(path)
            if not os.path.exists(path):
                result = RESULT_MISSING_DEST_DIR
        if result is RESULT_UNKNOWN:
            if json_file(filepath):
                version = json_file_ver(filepath) or 0 # replace None
                target = json_target_filepath(filepath, target)
                if not target:
                    result = RESULT_NO_JSON_TARGET_FILE
            else:
                version = 0
        if result is RESULT_UNKNOWN:
            if os.path.exists(target):
                match = file_compare_match(filepath, target)
                compare = match and 'Identical ' or 'Different '
                # Assume overwrite=True means overwrite even on match
                if overwrite:
                    action = "Overwrite"
                else:
                    if match:
                        result = RESULT_FILE_SKIPPED
                    else:
                        result = RESULT_FILE_MISMATCH
                    # ??? No db install table update - assume what is there is ok ??? (result=)
                    # XXX Actually what if the db doesn't have the history (pathological)
        if result is RESULT_UNKNOWN:
            cmd = ["cp", "-fp", quote(filepath), quote(target)]
            rc, err, out = system_command_output(cmd)
            if rc or err:
                result = RESULT_FILE_COPY_ERROR
                debug((cmd, rc, err, out))
            else:
                result = RESULT_SUCCESS
    except OSError:
        result = RESULT_EXCEPTION_OS_ERROR
    except IOError:
        result = RESULT_EXCEPTION_IO_ERROR
    except KeyboardInterrupt:
        result = RESULT_EXCEPTION_KEYBOARD
    except Exception:
        result = RESULT_EXCEPTION_UNHANDLED
        debug("copy_file unhandled exception: %s" % sys.exc_info()[1])
    finally:
        if result is RESULT_UNKNOWN:
            result = RESULT_FAIL
        rc = 1
        if result in [RESULT_SUCCESS]:
            rc = 0
            rstring, estring = "Passed", ""
        elif result in [RESULT_FILE_SKIPPED]:
            rc = 0
            rstring, estring = "Skipped", ""
        elif result in [RESULT_FILE_MISMATCH]:
            rstring, estring = "Skipped", ": %s" % get_result_string(result)
        else:
            rstring, estring = "Failed", ": %s" % get_result_string(result)
        msg = "%s %s: %s%s%s" % (action, rstring, compare, target, estring)
        rc and error(msg) or log(msg)
        if database and file_id is not None:
            database.update_install(file_id, result, db_time(), version)
    return rc

def copy_walk(database, pkg_id, loc_id, source, dest):
    rc = 0
    log("Installing %s to %s" % (source, dest))
    for dirpath, dirnames, filenames in os.walk(source):
        for filename in filenames:
            path = os.path.join(dirpath, filename)
            if copy_file(database, pkg_id, loc_id, source, dest, path):
                rc = 1
    return rc

def delete_dir(directory):
    err = None
    try:
        if not os.path.exists(directory):
            return None
        cmd = ["rm", "-rf", quote(directory)]
        rc, err, out = system_command_output(cmd)
        if rc or err:
            err = 'DeleteError'
    except OSError:
        err = 'OSError'
    except IOError:
        err = 'IOError'
    except KeyboardInterrupt:
        err = 'KeyboardInterrupt'
    except Exception:
        evalue = sys.exc_info()[1]
        err ='Unhandled Exception: %s' % evalue
    return err


##############################################################################
##  ___           _        _ _   _____ _ _           
## |_ _|_ __  ___| |_ __ _| | | |  ___(_) | ___  ___ 
##  | || '_ \/ __| __/ _` | | | | |_  | | |/ _ \/ __|
##  | || | | \__ \ || (_| | | | |  _| | | |  __/\__ \
## |___|_| |_|___/\__\__,_|_|_| |_|   |_|_|\___||___/
##                                                   
##############################################################################

class InstallFiles(object):

    def __init__(self, version, database_file, package, staging, locations,
                 options={}):
        self._version = version
        self._database_file = str(database_file)
        self._package = package
        self._staging = staging
        self._locations = locations
        self._options = options
        self._database = None
        self._package_id = None

    def options(self, key=None, default=None):
        if key is None:
            return self._options
        else:
            return self._options.get(key, default)

    def location_path(self, locations, location):
        path = get_location_path(locations, location)
        if isinstance(path, list):
            if self.options(LOCAL_FLAG):
                path = filter(not_ifs_path, path)
            if not path:
                path = None
            elif len(path) == 1:
                path = path[0]
        return path

    def open_database(self):
        if self._database_file == "None":
            return
        self._database = get_database(self._version, self._database_file,
                                      newfile='Auto', options=self._options)
        if not self._database:
            error("Unable to open database %s" % self._database_file)
            return
        self._database.open()
        self._package_id = self._database.package_id({
            "package": self._package, "result": RESULT_START,
            "count": 0, "time": db_time(),
            })

    def close_database(self, result=None):
        if self._database and self._package_id and result is not None:
            if DATABASE_UPDATE_PKG_INSTALL_TIME:
                timestamp = db_time()
            else:
                timestamp = None
            self._database.update_package(self._package_id, result, timestamp)
        self._database and self._database.close()
        self._database = None

    def get_location_id(self, locn, dest):
        if self._database:
            loc_id = self._database.location_id({
                "identifier": self._database.location_identifier(locn),
                "location": dest,
                })
        else:
            loc_id = None
        return loc_id

    # Don't use a generic os.walk for this function.
    # Only use os.walk in known staging src/dest matches.

    def install(self):
        rc, result = 0, RESULT_UNKNOWN
        try:
            if not check_tool_version(self._version):
                rc = 1
            if not rc:
                self.open_database()
                if not os.path.exists(self._staging):
                    rc, result = 1, RESULT_MISSING_STAGING_DIR
                    error("Staging dir %s does not exist" % self._staging)
                else:
                    rc = self.copy(self._staging, self._locations)
                    if rc:
                        result = RESULT_FAIL
                    else:
                        result = RESULT_SUCCESS
        finally:
            self.close_database(result)
        if self.options(LOCAL_FLAG):
            rc_sync = sync_msg = 0
        else:
            rc_sync = self.sync()
            sync_msg = ('Post-Install IFS File Sync: %s' %
                        (rc_sync and 'Failure' or 'Success'))
        log('Install Files Result: %s' % (rc and 'Failure' or 'Success'))
        sync_msg and log(sync_msg)
        return (rc or rc_sync) and 1 or 0

    def copy(self, staging, locations, locn=[]):
        rc = 0
        depth = len(locn)
        if not os.path.exists(staging):
            error("Staging dir %s does not exist" % staging)
            return 1
        isdir = lambda d: os.path.isdir(os.path.join(staging, d))
        staging_dirs = sorted(filter(isdir, os.listdir(staging)))
        self._options.update({'overwrite': False})
        for location in staging_dirs:
            if not location in locations:
                error("Unknown staging dir found: %s: NOT INSTALLED" % location)
                rc = 1
            else:
                nextlocn = locn + [location]
                sdir = os.path.join(staging, location)
                dest = self.location_path(locations, location)
                if dest is None:
                    continue
                if isinstance(dest, dict):
                    if self.copy(sdir, dest, nextlocn):
                        rc = 1
                elif isinstance(dest, basestring):
                    if self.walk(sdir, dest, nextlocn):
                        rc = 1
                elif isinstance(dest, list):
                    # Destination lists are [ifs, local] dest pairs.
                    # Do not need to install copy from staging to BOTH ifs
                    # and local, since we always end install() with a final
                    # sync(), cluster-wide.  Only do local copy walks.
                    for d in dest:
                        if not is_ifs_path(d):
                            if self.walk(sdir, d, nextlocn):
                                rc = 1
                else:
                    error("Invalid location %s" % dest)
                    rc = 1
        return rc

    def walk(self, sdir, dest, locn):
        rc = 0
        loc_id = self.get_location_id(locn, dest)
        is_ifs = is_ifs_path(dest)
        if is_ifs and self.options(LOCAL_FLAG):
            pass
        elif is_ifs and not ifs_mounted():
            error("/ifs is not mounted: Skipping %s installation to %s" %
                  (os.path.join(*locn), dest))
            rc = 1
            # XXX ??? Update DB for File for skip ??? (see copy_walk)
        else:
            lock = None
            if is_ifs_path(dest):
                lock = ifs_lock()
                if not lock:
                    error("UNABLE TO GET IFS LOCK TO INSTALL IFS FILES")
                    rc = 1
            try:
                if not rc:
                    if copy_walk(self._database, self._package_id, loc_id,
                                 sdir, dest):
                        rc = 1
            finally:
                lock and ifs_unlock()
        return rc

    def sync(self):
        # See comments regarding cluster sync (and isi_for_array) at top
        # We don't need an explicit ifs lock here, since sync_ifs_local does it
        log('Post-Install IFS file sync')
        rc = sync_ifs_local(self._locations, self._options)
        return rc

    def cleanup_staging(self):
        rc = 0
        if DELETE_STAGING_FILES:
            err = delete_dir(self._staging)
        else:
            err = None
        if err:
            error("%s: %s" % (err, self._staging))
            rc = 1
        return rc

    def cleanup(self):
        rc = 0
        self.close_database()
        if self.cleanup_staging():
            rc = 1
        return rc

    def __del__(self):
        self.cleanup()


def install_files(version, database_file, package, staging, locations,
                  options={}):
    install = InstallFiles(version, database_file, package, staging,
                           locations, options)
    return install.install()


##############################################################################
##  _____ _ _        ____  _        _     ___        __       
## |  ___(_) | ___  / ___|| |_ __ _| |_  |_ _|_ __  / _| ___  
## | |_  | | |/ _ \ \___ \| __/ _` | __|  | || '_ \| |_ / _ \ 
## |  _| | | |  __/  ___) | || (_| | |_   | || | | |  _| (_) |
## |_|   |_|_|\___| |____/ \__\__,_|\__| |___|_| |_|_|  \___/ 
##                                                            
##############################################################################

def file_hash(filename, htype=DEFAULT_HASH):
    hash = None
    htypelower = str(htype).lower()
    if htypelower == "none":
        pass
    elif htypelower in HASH_TYPES:
        cmd = [htypelower, "-q", quote(filename)]
        rc, err, out = system_command_output(cmd)
        if not rc and not err and out:
            hash = out[0]
        else:
            hash = "Error"
    else:
        raise Exception("Unsupported file hash type '%s'" % htype)
    return hash
    
def file_stat_mode_char(mode):
    modes = {
        stat.S_ISREG: '-',
        stat.S_ISBLK: 'b', 
        stat.S_ISCHR: 'c', 
        stat.S_ISDIR: 'd', 
        stat.S_ISLNK: 'l', 
        stat.S_ISFIFO: 'p',
        stat.S_ISSOCK: 's',
        }
    matches = filter(lambda m:m[0], [(m(mode), modes[m]) for m in modes])
    return len(matches) == 1 and matches[0][1] or '?'

def file_stat_mode_permissions(mode):
    umasks = stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR
    gmasks = stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP
    omasks = stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH
    perm = lambda m, masks: ''.join([
        (m & masks[0]) and 'r' or '-',
        (m & masks[1]) and 'w' or '-',
        (m & masks[2]) and 'x' or '-',
        ])
    return ''.join([perm(mode, masks) for masks in [umasks, gmasks, omasks]])

def file_stat_info(filename, directory=None, options={}):
    if not os.path.exists(filename):
        error("File %s does not exist" % filename)
        return None # To be filtered out in file_stat_info_sort
    fstat = os.stat(filename)
    fmode = (file_stat_mode_char(fstat.st_mode) +
             file_stat_mode_permissions(fstat.st_mode))
    flink = fstat.st_nlink
    fsize = fstat.st_size
    ftime = fstat.st_mtime
    if json_file(filename):
        model = json_file(filename)[0]
        fvers = json_file_ver(filename)
    else:
        model = ''
        fvers = ''
    overrides = options.get('overrides', {}).get(model, {})
    if overrides and isinstance(fvers, int):
        def listcheck(key):
            check = fvers in make_list(overrides.get(key, []))
            return check and key[0].upper() or ' '
        fover = ''.join(map(listcheck, ['blacklist', 'whitelist']))
    else:
        fover = ''
    if directory:
        dirreplace = directory.rstrip('/') + '/'
        fname = filename.replace(dirreplace, '')
    else:
        fname = filename
    if options.get("hash"):
        fhash = file_hash(filename, options.get("hash"))
    else:
        fhash = None
    return [fmode, flink, fsize, ftime, fvers, fover, fname, fhash]

file_stat_info_index = {
    'mode': 0,
    'link': 1,
    'size': 2,
    'time': 3, 'date': 3,
    'vers': 4, 'version': 4, 'ver': 4,
    'over': 5, 'override': 5,
    'name': 6,
    'hash': 7,
    }

def file_stat_info_sort(stats, key):
    stats = filter(None, stats) # Remove invalid filename cases
    index = file_stat_info_index.get(key)
    if index is not None:
        return sorted(stats, key=lambda s:s[index])
    return stats

def file_stat_time_format(fstat):
    nstat = fstat[:]
    index = file_stat_info_index['time']
    nstat[index] = time.strftime("%b %d %H:%M", time.localtime(fstat[index]))
    return nstat

def file_stat_format(fstat, widths=None, options={}):
    fstat = file_stat_time_format(fstat)
    if not widths:
       widths = [len(str(s)) for s in fstat]
    if not options.get("hash"):
        index = file_stat_info_index['hash']
        fstat = drop(fstat, index)
        widths = drop(widths, index)
    r = []
    [[r.append(x) for x in list(z)] for z in transpose([widths, fstat])]
    isnum = lambda s: type(s) in [int, long, float]
    format = ' '.join([isnum(s) and '%*s' or '%-*s' for s in fstat])
    return format % tuple(r)

def file_stat_widths(stats):
    wstats = filter(lambda s: isinstance(s, list), stats)
    wstats = map(file_stat_time_format, wstats)
    widths = data_widths(wstats)
    return widths


##############################################################################
##   ___                      _     _         ____ _               
##  / _ \__   _____ _ __ _ __(_) __| | ___   / ___| | __ _ ___ ___ 
## | | | \ \ / / _ \ '__| '__| |/ _` |/ _ \ | |   | |/ _` / __/ __|
## | |_| |\ V /  __/ |  | |  | | (_| |  __/ | |___| | (_| \__ \__ \
##  \___/  \_/ \___|_|  |_|  |_|\__,_|\___|  \____|_|\__,_|___/___/
##                                                                 
##############################################################################

class OverridesError(Exception):
    pass

class Overrides(object):

    def __init__(self, locations, options, filename, name,
                 replacement_chars):
        self._locations = locations
        self._options = options
        self._filename = filename
        self._name = name
        self._replacement_chars = replacement_chars
        self._overrides = {} # Cached, keyed by file location path

    def options(self, key=None, default=None):
        if key is None:
            return self._options
        else:
            return self._options.get(key, default)

    def location_path(self, location):
        path = get_location_path(self._locations, location)
        if isinstance(path, list):
            if self.options(LOCAL_FLAG):
                path = filter(not_ifs_path, path)
            if not path:
                path = None
            elif len(path) == 1:
                path = path[0]
        return path

    def overrides(self, location):
        # location might be a full path already depending on caller
        if os.path.isabs(location):
            path = location
        else:
            path = self.location_path(location)
            # path might be a list of paths for given location
        if isinstance(path, list):
            if self.options(LOCAL_FLAG):
                path = filter(not_ifs_path, path)
            for p in path:
                if p and not p in self._overrides:
                    self._overrides[p] = self.load(p)
            overrides = [self._overrides.get(p, {}) for p in path]
        else:
            if path and not path in self._overrides:
                self._overrides[path] = self.load(path)
            overrides = self._overrides.get(path, {})
        return path, overrides

    def overrides_ifs_local(self, location):
        # Returns dict {'ifs':overrides, 'local':overrides}
        paths = self.location_path(location)
        if not isinstance(paths, list):
            paths = [paths]
        overrides = {}
        for path, override in dict(map(self.overrides, paths)).iteritems():
            if is_ifs_path(path):
                overrides['ifs'] = (path, override)
            else:
                overrides['local'] = (path, override)
        return overrides

    def skip(self, msg):
        error("%s: SKIPPED" % msg)

    def string_safe(self, over):
        return not reduce(lambda a, b: a or b,
                          [r in str(over) for r in self._replacement_chars])

    def subclass_implementation_error(self):
        raise OverridesError(
            "Function %s must be implemented in subclass" % CallerName())

    def _load(self, path):
        self.subclass_implementation_error()

    def load(self, path):
        # If not exists, quietly return nothing
        if isinstance(path, basestring):
            return self._load(path)
        elif isinstance(path, list):
            return map(self._load, path)
        else:
            error("Invalid overrides path: %s" % path)
            return {}

    def _save(self, path, overrides):
        self.subclass_implementation_error()

    def save(self, path, overrides):
        self._overrides = {} # Clear cache on saves
        if isinstance(path, basestring):
            rc = self._save(path, overrides)
        elif isinstance(path, list):
            rc = reduce(lambda a, b: a or b,
                        [self._save(p, overrides) for p in path])
        else:
            error("Invalid save path type %s" % type_str(path))
            rc = 1
        return rc

    # Currently, taking a bidirectional "add always" merge approach:
    # If a (model, key, value) does not exist in either ifs or local:
    # => ALWAYS add the missing info to the other
    # => NEVER "remove" info from either
    # So if a node joins a cluster with "extra" info, this will be
    # merged and synced to the cluster .. and vice versa.
    # The only way for info to be removed is via explicit clearing.

    def merge_ifs_local_dicts(self, overrides):
        result = {}
        rmdup = lambda data: sorted(remove_duplicates(flatten(data)))
        models = rmdup([over.keys() for path, over in overrides.values()])
        def get_model_override(model, key):
            return key in overrides and overrides[key][1].get(model) or None
        def get_model_overrides(model):
            return [get_model_override(model, key) for key in ['ifs','local']]
        for model in models:
            ifs, local = get_model_overrides(model)
            if ifs and local:
                keys = rmdup([override.keys() for override in [ifs, local]])
                temp = {}
                for key in keys:
                    value = rmdup([make_list(override.get(key, []))
                                   for override in [ifs, local]])
                    if len(value) == 1:
                        value = value[0]
                    temp[key] = value
                result[model] = temp
            elif ifs:
                result[model] = ifs
            else:
                result[model] = local
        return result

    def overrides_merged(self, location):
        overrides = self.overrides_ifs_local(location)
        if not overrides:
            error("Invalid merge overrides for location %s" % location)
            return None
        location_keys = ['ifs', 'local']
        if self.options(LOCAL_FLAG):
            location_keys = ['local']
        for key in location_keys:
            if overrides.get(key) is None:
                error("Invalid merge, expected one %s path (none found)" % key)
                return None
        return self.merge_ifs_local_dicts(overrides)

    def _sync(self, location):
        rc = 0
        overrides = self.overrides_ifs_local(location)
        if not overrides or not 'ifs' in overrides:
            error("Location %s does not support syncing" % location)
            return 1
        # Previously we used merged (overrides_merged) data in sync,
        # saving the merged data to both ifs and local. This had
        # problems with override DELETIONS not properly propagating.
        # They would in fact be undone on a sync (See comments above,
        # re "add always" merge approach).
        # We now only COPY, FROM ifs ("master") TO local node
        # files, and never update ifs using (merged) local data. 
        # Any (forced) local-only changes will not survive a sync.
        # Get ifs override data, then save to local override path.
        key = 'ifs'
        temp, override = overrides.get(key)
        path, temp = overrides.get('local')
        # if no override, then we have no data - write new file
        debug("Saving synced %s overrides data to %s" % (key, path))
        if self.save(path, override):
            rc = 1
        return rc

    def sync(self, location):
        # Need to lock ifs for entire sync() operation -- don't want to
        # do a sync using stale ifs overrides data from initial read,
        # then writing stale merged data back to ifs post-sync merge.
        rc, lock = 0, None
        if self.options(LOCAL_FLAG):
            return rc
        try:
            lock = ifs_lock()
            rc = self._sync(location)
        finally:
            lock and ifs_unlock()
        return rc

    def _add(self, model, key, value, overrides):
        self.subclass_implementation_error()

    def _remove(self, model, key, value, overrides):
        self.subclass_implementation_error()

    def _delete(self, model, key, value, overrides):
        self.subclass_implementation_error()

    def _change_check_value(self, change, location, model, key, value):
        self.subclass_implementation_error()

    def _change_check_newvalue(self, change, location, model, key, value,
                               newvalue):
        self.subclass_implementation_error()

    def _change(self, change, location, model, key, value, func):
        rc = 0
        # See comments in _sync(). Previously we would use merged 
        # overrides (overrides_merged) data here in _change, but now
        # we only use ifs "master" source data -- UNLESS forced_local.
        overrides = self.overrides_ifs_local(location)
        if self.options(LOCAL_FLAG):
            path, overrides = overrides.get('local')
        else:
            path, overrides = overrides.get('ifs')
        if model not in overrides:
            if change in ['add']:
                log("Drive model %s added to %s" % (model, self._name))
                overrides[model] = {}
            elif change in ['remove', 'delete']:
                log("Drive model %s is not in %s" % (model, self._name))
                return 0
        over = overrides.get(model, {})
        value = self._change_check_value(change, location, model, key, value)
        msg, newvalue = func(model, key, value, over)
        msg and log(msg)
        if self._change_check_newvalue(change, location, model, key, value,
                                       newvalue):
            rc = 1
        if newvalue is not False:
            if newvalue == ('delete',):
                if key is None and change == 'delete':
                    del overrides[model]
                elif key in over:
                    del over[key]
                    if not over:
                        del overrides[model]
            else:
                over.update({key: newvalue})
            if self.save(self.location_path(location), overrides):
                # Errors are already reported in code
                rc = 1
            if self.options(LOCAL_FLAG):
                warn("Override changes made in %s mode are TEMPORARY:" % LOCAL_FLAG)
                warn("Changes will be LOST on next ifs sync.")
        return rc

    def change(self, change, location, model, key=None, value=None):
        # Need to lock ifs for entire change() operation -- simply locking
        # the updated ifs file write is insufficient if multiple nodes are
        # attempting to change() the overrides file (data merging/conflicts)
        rc, lock = 0, None
        func = {
            'add': self._add,
            'remove': self._remove,
            'delete': self._delete,
            }.get(change)
        if not func:
            error("Unknown %s change action: %s" % (self._name, change))
            return 1
        try:
            # XXX What about forced_local ???
            lock = ifs_lock()
            rc = self._change(change, location, model, key, value, func)
        except OverridesError as e:
            error(str(e))
            rc = 1
        finally:
            lock and ifs_unlock()
        return rc

    def add(self, location, model, key=None, value=None):
        return self.change('add', location, model, key, value)

    def remove(self, location, model, key=None, value=None):
        return self.change('remove', location, model, key, value)

    def delete(self, location, model, key=None):
        return self.change('delete', location, model, key)

    def _display(self, path, overrides):
        self.subclass_implementation_error()

    def display_overrides(self, paths, overrides):
        rc = 0
        if isinstance(overrides, dict):
            rc = self._display(paths, overrides)
        elif isinstance(overrides, list):
            rc = reduce(lambda a, b: a or b,
                        [self._display(path, over)
                         for path, over in zip(paths, overrides)])
        else:
            error("Invalid overrides data type %s" % type_str(overrides))
            rc = 1
        return rc

    def display(self, location):
        # Before loading and displaying data, first sync
        rc = self.sync(location)
        if rc:
            error("Error syncing ifs and local %s" % self._name)
        overpath, overrides = self.overrides(location)
        if not filter(None, overrides):
            print("No %s set" % self._name)
        else:
            print("%s %s:" % (location, self._name))
            if self.display_overrides(overpath, overrides):
                rc = 1
        return rc


##############################################################################
##   ___                      _     _       __     __            _             
##  / _ \__   _____ _ __ _ __(_) __| | ___  \ \   / /__ _ __ ___(_) ___  _ __  
## | | | \ \ / / _ \ '__| '__| |/ _` |/ _ \  \ \ / / _ \ '__/ __| |/ _ \| '_ \ 
## | |_| |\ V /  __/ |  | |  | | (_| |  __/   \ V /  __/ |  \__ \ | (_) | | | |
##  \___/  \_/ \___|_|  |_|  |_|\__,_|\___|    \_/ \___|_|  |___/_|\___/|_| |_|
##                                                                             
##############################################################################

# Version Override file format:
#  1. one drive model per line
#  2. optionally quoted, but unnecessary
#  3. Whitespace matters: model name includes everything from the 
#     first column, all way to the first field delimiter ":"
#  4. colon ":" separated field:values
#  5. CSV comma separated values
#  6. semicolon ";" separated multiple field:values
#     (Optionally terminating all, including last one)
#  7. blacklist values are integer versions; multiple allowed
#  8. whitelist values are integer versions; multiple not allowed
#  9. If we have fields later with values containing commas,
#     a) values could be quoted (double quotes)
#     b) commas could be escaped in the values with: \,
#     But this is not necessary at this time
# 10. Whitespace does not matter in fields and values
#     a) For fields, use strip() to remove leading/trailing whitespace
#     b) For values, expect integer values only, at this time
# 
# Version Override code format: 
# 1. use dict, with keys=model string, values=dict of fields:values
# 2. blacklist is either Integer, or List of Integers
# 3. whitelist is always Integer
# 
# Version Override conversion:
# 1. Always save models in sorted order for reproducibility
# 2. Always save fields in sorted order for reproducibility
# 3. Always save values in sorted order for reproducibility

class VersionOverrides(Overrides):

    VALUE_FIELD_SEP = ":"
    MULTI_FIELD_SEP = ";"
    VALUE_SEP = ","

    def __init__(self, locations, options, filename=OVERRIDE_VERS_FILENAME):
        name = "file version overrides"
        replacement_characters = [
            self.VALUE_FIELD_SEP, self.MULTI_FIELD_SEP, self.VALUE_SEP,
            ]
        Overrides.__init__(self, locations, options, filename, name,
                           replacement_characters)

    def _dict_to_array(self, overrides):
        over = []
        if not isinstance(overrides, dict):
            error("Invalid overrides data (expected dict)")
            return over
        for model in sorted(overrides.keys()):
            if not self.string_safe(model):
                self.skip('Model "%s" contains illegal characters' % model)
                continue
            data = []
            for field in sorted(overrides[model].keys()):
                value = overrides[model][field]
                if isinstance(value, int):
                    vdata = str(value)
                elif isinstance(value, list):
                    vdata = sorted(remove_duplicates(value))
                    vdata = self.VALUE_SEP.join(map(str, vdata))
                else:
                    vdata = str(value) # INVALID, really...
                data.append("".join((field, self.VALUE_FIELD_SEP, vdata)))
            over.append("".join((model, self.VALUE_FIELD_SEP,
                                 self.MULTI_FIELD_SEP.join(data))))
        return over

    def _array_to_dict(self, overrides):
        over = {}
        if not isinstance(overrides, list):
            error("Invalid overrides data (expected list)")
            return over
        invalid = lambda msg, model, check: self.skip(
            "Invalid data: %s: %s: %s" %
            (model, msg, ", ".join(map(str, check))))
        for line in overrides:
            ss = line.rstrip().split(self.VALUE_FIELD_SEP)
            model = ss[0]
            ff = self.VALUE_FIELD_SEP.join(ss[1:])
            ff = ff.strip(self.MULTI_FIELD_SEP)
            if not ff:
                data = {}
            else:
                ff = ff.split(self.MULTI_FIELD_SEP)
                vv = [f.split(self.VALUE_FIELD_SEP) for f in ff]
                check = filter(lambda v: len(v)!=2, vv)
                if check:
                    invalid("Improper field:value format", model, check)
                    vv = filter(lambda v: len(v)==2, vv)
                check = filter(lambda v: not v[0], vv)
                if check:
                    invalid("Missing field", model, check)
                    vv = filter(lambda v: v[0], vv)
                check = filter(lambda v: not v[1], vv)
                if check:
                    invalid("Missing value", model, check)
                    vv = filter(lambda v: v[1], vv)
                data = dict(vv)
            for field, vdata in data.iteritems():
                try:
                    if self.VALUE_SEP in vdata:
                        value = map(int, vdata.split(self.VALUE_SEP))
                    else:
                        value = int(vdata)
                except ValueError:
                    value = str(vdata) # INVALID, really...
                if not value:
                    self.skip("No valid override data for %s" % model)
                    continue
                data[field] = value
            if not data:
                self.skip("No valid override data for %s" % model)
                continue
            if model in over:
                warn('%s is already in override dictionary: REPLACING' %
                     model)
            over[model] = data
        return over

    def _load(self, path):
        # If not exists, quietly return nothing
        overrides = {}
        path = os.path.join(path, self._filename)
        if os.path.exists(path):
            data = ifs_safe_file_read(path)
            if data:
                overrides = self._array_to_dict(data)
        return overrides

    def _save(self, path, overrides):
        if not isinstance(path, basestring):
            error("Invalid save path type %s" % type_str(path))
            return 1
        data = self._dict_to_array(overrides)
        filename = os.path.join(path, self._filename)
        rc = ifs_safe_file_write(filename, data)
        if rc:
            error("Save overrides data failed")
        return rc

    def _add(self, model, key, value, overrides):
        name = "%s version %s" % (model, value)
        action = "Adding %s %s" % (key, name)
        already = "%s is already in %s" % (name, key)
        previous = make_list(overrides.get(key, []))
        if isinstance(previous, int):
            if value == previous:
                msg, newvalue = already, False
            else:
                msg, newvalue = action, sorted([previous, value])
        elif isinstance(previous, list):
            if value in previous:
                msg, newvalue = already, False
            else:
                msg = action
                newvalue = sorted(remove_duplicates(previous + [value]))
        else:
            warn("Invalid %s %s type %s: Replacing existing %s" %
                 (model, key, type_str(previous), previous))
            msg, newvalue = action, value
        return msg, newvalue

    def _remove(self, model, key, value, overrides):
        name = "%s version %s" % (model, value)
        action = "Removing %s %s" % (key, name)
        already = "%s is already removed from %s" % (name, key)
        previous = make_list(overrides.get(key, []))
        if isinstance(previous, int):
            if value == previous:
                msg, newvalue = action, ('delete',)
            else:
                msg, newvalue = already, False
        elif isinstance(previous, list):
            if previous == [value]:
                msg, newvalue = action, ('delete',)
            elif value in previous:
                msg = action
                newvalue = sorted(remove_duplicates(remove(previous, value)))
            else:
                msg, newvalue = already, False
        else:
            warn("Invalid %s %s type %s: Removing existing %s" %
                 (model, key, type_str(previous), previous))
            msg, newvalue = action, ('delete',)
        return msg, newvalue

    def _delete(self, model, key, value, overrides):
        if overrides and key in overrides:
            msg = "Deleting %s %s %s" % (model, key, self._name)
            newvalue = ('delete',)
        else:
            msg = "%s %s is already removed from %s" % (model, key, self._name)
            newvalue = False
        return msg, newvalue


    def _change_check_value(self, change, location, model, key, value):
        posintkeys = ['blacklist', 'whitelist']
        if (not change in ['delete'] and key in posintkeys and
            (not isinstance(value, int) or not value > 0)):
            err = ("Invalid %s version: %s" % (key, value))
            raise OverridesError(err)
        return value

    def _change_check_newvalue(self, change, location, model, key, value,
                               newvalue):
        if change in ['add'] and newvalue is not False:
            overrides = self.overrides_merged(location)
            over = overrides.get(model, {})
            if over is None: # E.g. overrides = {'Model Foo': None}
                over = {}
            otherkeys = {'blacklist':'whitelist', 'whitelist':'blacklist'}
            otherkey = otherkeys.get(key)
            otherlist = make_list(over.get(otherkey, []))
            if newvalue is not None and otherlist and value in otherlist:
                name = "%s version %s" % (model, value)
                warn("New %s %s is also in %s" % (key, name, otherkey))
        return 0

    def blacklisted(self, location, model, version):
        """Returns None if there is no model blacklist defined,
        True  if the given model,version is in the model blacklist,
        False if the given model,version is NOT in the model blacklist"""
        overrides = self.overrides_merged(location)
        blacklist = overrides.get(model, {}).get('blacklist')
        if blacklist is None:
            return None
        return version in make_list(blacklist)

    def whitelisted(self, location, model, version):
        """Returns None if there is no model whitelist defined,
        True  if the given model,version is in the model whitelist,
        False if the given model,version is NOT in the model whitelist"""
        overrides = self.overrides_merged(location)
        whitelist = overrides.get(model, {}).get('whitelist')
        if whitelist is None:
            return None
        return version in make_list(whitelist)

    def _display(self, path, overrides):
        if not isinstance(overrides, dict):
            critical_error("Invalid %s data type %s" %
                           (self._name, type_str(overrides)))
            return 1
        models = sorted(overrides.keys())
        keys = flatten([v.keys() for v in overrides.values()])
        keys = sorted(remove_duplicates(keys))
        confkeys = ['blacklist', 'whitelist']
        confvalf = lambda v: set(isinstance(v, int) and [v] or
                                 isinstance(v, list) and v or [])
        conflicts, data = {}, []
        for model in models:
            overs = overrides[model]
            confdata = [confvalf(overs.get(key)) for key in confkeys]
            conflicts[model] = list(reduce(lambda a, b: a & b, confdata))
            overdata = [','.join(map(str, sorted(make_list(d))))
                        for d in [overs.get(key, '') for key in keys]]
            data.append([model] + overdata)
        if flatten(conflicts.values()):
            keys.append('conflicts')
            for d in data:
                conflict = conflicts[d[0]]
                d.append(conflict and ','.join(sorted(map(str, conflict))) or '')
        if data:
            print("")
            print("%s:" % path)
            headers = [h.capitalize() for h in ['model'] + keys]
            data_dump(data, headers, self.options())
        return 0


##############################################################################
##   ___                      _     _        ____        _        
##  / _ \__   _____ _ __ _ __(_) __| | ___  |  _ \  __ _| |_ __ _ 
## | | | \ \ / / _ \ '__| '__| |/ _` |/ _ \ | | | |/ _` | __/ _` |
## | |_| |\ V /  __/ |  | |  | | (_| |  __/ | |_| | (_| | || (_| |
##  \___/  \_/ \___|_|  |_|  |_|\__,_|\___| |____/ \__,_|\__\__,_|
##                                                                
##############################################################################

class DataOverrides(Overrides):

    KEYSPEC_FIELD_SEP = ":"

    DELETE_IDENT = None
    DELETE_TYPE = type(DELETE_IDENT)

    # Currently, we only support simple elements: numbers, strings.
    # No complex elements (lists, dicts) are supported at this time.

    VALUE_TYPES = ['number', 'string', 'delete']
    VALUE_TYPES_SUPPORTED = [
        int, float, basestring, str, unicode, DELETE_TYPE,
        ]

    def valid_value_type(self, value):
        return type(value) in self.VALUE_TYPES_SUPPORTED

    def __init__(self, locations, options, filename=OVERRIDE_DATA_FILENAME):
        name = "data value overrides"
        replacement_characters = [
            self.KEYSPEC_FIELD_SEP,
            ]
        Overrides.__init__(self, locations, options, filename, name,
                           replacement_characters)

    def _load(self, path):
        # WE ASSUME THAT CALLER HAS ALREADY LOCKED IFS AS NECESSARY
        # If not exists, quietly return nothing
        overrides = {}
        path = os.path.join(path, self._filename)
        if os.path.exists(path):
            try:
                with open(path, 'r') as f:
                    overrides = pickle.load(f)
            except:
                error("Error reading %s overrides file" % location)
                overrides = {}
            if not overrides:
                pass # Don't bother warning if none loaded
        return overrides

    def _save(self, path, overrides):
        # WE ASSUME THAT CALLER HAS ALREADY LOCKED IFS AS NECESSARY
        rc = 0
        if not isinstance(path, basestring):
            error("Invalid save path type %s" % type_str(path))
            return 1
        filename = os.path.join(path, self._filename)
        dirname = os.path.dirname(filename)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        if not os.path.exists(dirname):
            error("Unable to create override directory %s" % dirname)
            rc = 1
        else:
            try:
                with open(filename, 'w') as f:
                    pickle.dump(overrides, f)
            except:
                error("Error writing overrides file %s" % filename)
                rc = 1
        return rc

    def _add(self, model, key, value, overrides):
        if self.valid_value_type(value):
            valstr = "%s(%s)" % (type_str(value), value)
            name = "%s %s %s" % (model, key, valstr)
            previous = overrides.get(key)
            if key not in overrides:
                msg = "Adding %s" % (name)
                newvalue = value
            elif previous == value and type(previous) == type(value):
                msg = "%s is already in %s" % (name, self._name)
                newvalue = False
            else:
                msg = ("%s %s was %s(%s), now %s" %
                       (model, key, type_str(previous), previous, valstr))
                newvalue = value
        else:
            error("Invalid %s data override value type %s" %
                  (model, type_str(value)))
            msg, newvalue = "Override ABORTED", False
        return msg, newvalue

    def _remove(self, model, key, value, overrides):
        name = "%s %s" % (model, key)
        if not key in overrides:
            msg = "%s is already removed from %s" % (name, self._name)
            newvalue = False
        else:
            previous = overrides.get(key)
            valstr = "%s(%s)" % (type_str(previous), previous)
            msg = "Removing %s %s" % (name, valstr)
            newvalue = ('delete',)
        return msg, newvalue

    def _delete(self, model, key, value, overrides):
        if overrides:
            msg = "Deleting %s %s" % (model, self._name)
            newvalue = ('delete',)
        else:
            msg, newvalue = None, False
        return msg, newvalue

    def _change_check_value(self, change, location, model, key, value):
        if value is None and change == 'add':
            values = dict(filter(lambda v: v[1] is not None,
                                 [(v, self.options(v))
                                  for v in self.VALUE_TYPES]))
            if len(values) == 0:
                err = ("Missing required %s value" % self._name)
                raise OverridesError(err)
            elif len(values) > 1:
                values = ", ".join(sorted(values.keys()))
                err = ("Too many %s values: %s" % (self._name, values))
                raise OverridesError(err)
            if values:
                valtype, value = values.items()[0]
                if valtype == 'delete' and value is True:
                    value = self.DELETE_IDENT
            else:
                valtype = value = None
        return value

    def _change_check_newvalue(self, change, location, model, key, value,
                               newvalue):
        return 0

    def _display(self, path, overrides):
        if not isinstance(overrides, dict):
            critical_error("Invalid %s data type %s" %
                           (self._name, type_str(overrides)))
            return 1
        data, headers = [], ["Model", "Key Spec", "Type", "Override"]
        models = sorted(overrides.keys())
        for model in models:
            first = model
            for key in sorted(overrides[model].keys()):
                val = overrides[model][key]
                if val == self.DELETE_IDENT:
                    typ, value = "", "DELETE"
                else:
                    typ, value = (type_str(val), val)
                mod, first = first and (model, None) or ("", None)
                data.append([mod, key, typ, value])
        if data:
            print("")
            print("%s:" % path)
            data_dump(data, headers, self.options())
        return 0


##############################################################################
##   ___                      _     _         ____               _     
##  / _ \__   _____ _ __ _ __(_) __| | ___   / ___|_ __ ___   __| |___ 
## | | | \ \ / / _ \ '__| '__| |/ _` |/ _ \ | |   | '_ ` _ \ / _` / __|
## | |_| |\ V /  __/ |  | |  | | (_| |  __/ | |___| | | | | | (_| \__ \
##  \___/  \_/ \___|_|  |_|  |_|\__,_|\___|  \____|_| |_| |_|\__,_|___/
##                                                                     
##############################################################################

def blacklist_add(locations, model, version, options={}):
    location, key = 'config/drives', 'blacklist'
    overrides = VersionOverrides(locations, options)
    return overrides.add(location, model, key, version)

def blacklist_remove(locations, model, version, options={}):
    location, key = 'config/drives', 'blacklist'
    overrides = VersionOverrides(locations, options)
    return overrides.remove(location, model, key, version)

def blacklist_delete(locations, model, options={}):
    location, key = 'config/drives', 'blacklist'
    overrides = VersionOverrides(locations, options)
    return overrides.delete(location, model, key)

def whitelist_add(locations, model, version, options={}):
    location, key = 'config/drives', 'whitelist'
    overrides = VersionOverrides(locations, options)
    return overrides.add(location, model, key, version)

def whitelist_remove(locations, model, version, options={}):
    location, key = 'config/drives', 'whitelist'
    overrides = VersionOverrides(locations, options)
    return overrides.remove(location, model, key, version)

def whitelist_delete(locations, model, options={}):
    location, key = 'config/drives', 'whitelist'
    overrides = VersionOverrides(locations, options)
    return overrides.delete(location, model, key)

def data_override_add(locations, model, key, options={}):
    location = 'config/drives'
    overrides = DataOverrides(locations, options)
    return overrides.add(location, model, key)

def data_override_remove(locations, model, key, options={}):
    location = 'config/drives'
    overrides = DataOverrides(locations, options)
    return overrides.remove(location, model, key)

def data_override_delete(locations, model, options={}):
    location = 'config/drives'
    overrides = DataOverrides(locations, options)
    return overrides.delete(location, model)

def overrides_display(locations, options={}):
    rc = 0
    location = 'config/drives'
    overrides = VersionOverrides(locations, options)
    if overrides.display(location):
        rc = 1
    overrides = DataOverrides(locations, options)
    if overrides.display(location):
        rc = 1
    return rc


##############################################################################
##  ____       _           _   ____  _        _       
## / ___|  ___| | ___  ___| |_/ ___|| |_ __ _| |_ ___ 
## \___ \ / _ \ |/ _ \/ __| __\___ \| __/ _` | __/ _ \
##  ___) |  __/ |  __/ (__| |_ ___) | || (_| | ||  __/
## |____/ \___|_|\___|\___|\__|____/ \__\__,_|\__\___|
##                                                    
##############################################################################

class SelectState(object):
    # We only track state locally, NOT on ifs. If we ever did read/write
    # state files on ifs, we would need to lock ifs to be totally safe.

    def __init__(self, locations, options, filename=SELECT_STATE_FILENAME):
        self._locations = locations
        self._options = options
        self._filename = filename
        self._state = {}

    def options(self, key=None, default=None):
        if key is None:
            return self._options
        else:
            return self._options.get(key, default)

    def location_path(self, location):
        path = get_location_path(self._locations, location)
        if isinstance(path, list):
            path = filter(not_ifs_path, path)
            path = path and path[0] or None
        if path:
            path = os.path.join(path, self._filename)
        return path 

    def read(self, location):
        state = {}
        statefile = self.location_path(location)
        if statefile:
            if os.path.exists(statefile):
                try:
                    with open(statefile, 'r') as statef:
                        state = pickle.load(statef)
                except:
                    error("Error reading %s state file" % location)
                    state = {}
        self._state[location] = state

    def write(self, location):
        if not self._state:
            error("No state information available to write")
        elif not location in self._state:
            error("No %s state information available to write")
        else:
            state = self._state.get(location)
            statefile = self.location_path(location)
            statepath = os.path.dirname(statefile)
            if not os.path.exists(statepath):
                os.makedirs(statepath)
            if not os.path.exists(statepath):
                error("Unable to create %s state directory %s" %
                      (location, statepath))
            else:
                try:
                    with open(statefile, 'w') as statef:
                        pickle.dump(state, statef)
                except:
                    error("Error writing %s state file" % location)

    def select(self, location, filespec):
        select = None
        if not location in self._state:
            self.read(location)
        if location in self._state:
            selected = self._state.get(location)
            if not isinstance(selected, dict):
                error("Invalid SelectState dict for %s" % location)
            else:
                select = selected.get(filespec)
        return select

    def update(self, location, filespec, filepath):
        if not location in self._state:
            self.read(location)
        if not location in self._state:
            self._state[location] = {}
        self._state[location].update({filespec: filepath})
        self.write(location)

    def missing(self, location, filespec):
        if not location in self._state:
            self.read(location)
        missings = self._state.get(location, {}).get('MISSING', {})
        return missings.get(filespec, False)

    def update_missing(self, location, filespec, missing):
        if not location in self._state:
            self.read(location)
        if not location in self._state:
            self._state[location] = {}
        if not 'MISSING' in self._state[location]:
            self._state[location].update({'MISSING': {}})
        self._state[location]['MISSING'].update({filespec: missing})
        self.write(location)

    def set_missing(self, location, filespec):
        return self.update_missing(location, filespec, True)

    def clear_missing(self, location, filespec):
        return self.update_missing(location, filespec, False)

    # We could have a close() which __del__ calls, which calls
    # self.write(location) for each location in self._state, but this
    # is not necessary if we do a write() on every update()


##############################################################################
##  ___           _        _ _          _   _____ _ _           
## |_ _|_ __  ___| |_ __ _| | | ___  __| | |  ___(_) | ___  ___ 
##  | || '_ \/ __| __/ _` | | |/ _ \/ _` | | |_  | | |/ _ \/ __|
##  | || | | \__ \ || (_| | | |  __/ (_| | |  _| | | |  __/\__ \
## |___|_| |_|___/\__\__,_|_|_|\___|\__,_| |_|   |_|_|\___||___/
##                                                              
##############################################################################

class InstalledFiles(object):

    def __init__(self, locations, options={}):
        self._locations = locations
        self._location = 'config/drives'
        self._options = options
        self._files = self._dict = None
        self._overrides = {
            'version': VersionOverrides(locations, options),
            'data': DataOverrides(locations, options),
            }
        self._selectState = SelectState(locations, options)
        self._selected = {}
        self.sync()

    def options(self, key=None, default=None):
        if key is None:
            return self._options
        else:
            return self._options.get(key, default)

    def location_path(self, location):
        path = get_location_path(self._locations, location)
        if isinstance(path, list):
            if self.options(LOCAL_FLAG):
                path = filter(not_ifs_path, path)
            if not path:
                path = None
            elif len(path) == 1:
                path = path[0]
        return path

    def sync(self):
        # We don't need an explicit ifs lock here, since sync_ifs_local does it
        debug('IFS file sync')
        rc = 0
        # sync_ifs_local code can handle combination of ifs not mounted and
        # WARN_IF_IFS_UNMOUNTED==True, but pre-emptively skip to minimize
        # code execution and message logging.
        if WARN_IF_IFS_UNMOUNTED and not ifs_mounted():
            warn("Skipping sync of ifs and local dirs: ifs is not mounted")
        else:
            if sync_ifs_local(self._locations, self._options):
                rc = 1
            if self._overrides['version'].sync(self._location):
                rc = 1
            if self._overrides['data'].sync(self._location):
                rc = 1
        return rc

    def overrides(self, override_type, location):
        if not override_type in self._overrides:
            raise Exception("Invalid override type %s" % override_type)
        overpath, overrides = self._overrides[override_type].overrides(location)
        if isinstance(overrides, list):
            return dict(zip(overpath, overrides))
        else:
            return {overpath: overrides}

    def apply_data_overrides(self, location, filename):
        if not filename or not os.path.exists(filename):
            return None
        overrides = self.overrides('data', location)
        if not overrides:
            result = ifs_safe_file_read(filename)
            return result
        # Overrides should be a dict of ifs and local results
        # Take either, should be identical after sync
        # XXX We can clean this up later, searching for non-ifs local
        # path, and verify at least one, and taking [0]th of this
        overridepathkey = overrides.keys()[0]
        overrides = overrides.get(overridepathkey, {})
        # Determine drive model from filename.
        # If it's a valid JSON file, we can use our regex - else failure
        match = json_file(filename)
        if match:
            model, ver = match
        else:
            error("Invalid data file: Not a json filename: %s" % filename)
            return None
        models = overrides.keys()
        # Later, if we want, we can use regex patterns in data overrides
        # model strings, & change this simple "model in models" code
        # But that might be opening a whole can of worms (bugs)
        if not model in models:
            result = ifs_safe_file_read(filename)
            return result
        overrides = overrides.get(model, {})
        jobj = json_file_data(filename)
        if not jobj:
            return None
        keyspecs = sorted(overrides.keys())
        for keyspec in keyspecs:
            newvalue = overrides[keyspec]
            jptr = jobj
            # XXX FIX THIS LATER use class somehow; self.KEYSPEC_FIELD_SEP
            keys = str(keyspec).split(':') # We need at least one ...
            if not keys:
                continue # XXX ??? error?
            for key in keys[:-1]:
                if not key in jptr:
                    jptr[key] = {}
                jptr = jptr[key]
            key = keys[-1]
            if isinstance(jptr, dict):
                jptr[key] = newvalue
            else:
                # Use a warn for override complaints (report to stderr,
                # but do not return a non-zero error code return value)
                warn("Unable to apply %s override %s = %s" %
                     (model, keyspec, newvalue))
        result = json.dumps(jobj, sort_keys=False, indent=4)
        return result

    def blacklisted(self, location, model, version):
        return self._overrides['version'].blacklisted(location, model, version)

    def whitelisted(self, location, model, version):
        return self._overrides['version'].whitelisted(location, model, version)

    def _files_walk(self, destination):
        options = self.options()
        files = []
        filestat = lambda p: file_stat_info(p, destination, options=options)
        for dirpath, dirnames, filenames in os.walk(destination):
            filenames = filter_installed_files(filenames)
            filepaths = [os.path.join(dirpath, name) for name in filenames]
            files.extend(map(filestat, filepaths))
        return file_stat_info_sort(files, key=options.get("sort"))

    def _files_find(self, dest, locn):
        files = []
        if is_ifs_path(dest) and not ifs_mounted():
            files.append(('skip', locn, dest))
            if not WARN_IF_IFS_UNMOUNTED:
                error("/ifs is not mounted: Skipping %s" % dest)
            # XXX ??? Do something else? To get into db history?
        else:
            overrides = self.overrides('version', dest)
            if overrides:
                self._options['overrides'] = overrides.get(dest, {})
            else:
                self._options['overrides'] = {}
            installed = self._files_walk(dest)
            if installed:
                files.append(('walk', locn, dest))
                files.extend(installed)
        return files

    def files(self, locations=None, locn=[]):
        if self._files is not None:
            return self._files
        if locations is None:
            locations = self._locations
        files = []
        for location in sorted(locations.keys()):
            nextlocn = locn + [location]
            dest = self.location_path(location)
            if not dest:
                dest = self.location_path(os.path.join(*nextlocn))
            if dest is None:
                continue
            if isinstance(dest, dict):
                files.append(('dict', nextlocn, dest))
                files.extend(self.files(dest, nextlocn))
            elif isinstance(dest, basestring):
                files.extend(self._files_find(dest, nextlocn))
            elif isinstance(dest, list):
                for d in dest:
                    files.extend(self._files_find(d, nextlocn))
            else:
                error("Invalid location %s" % dest)
        self._files = files
        return self._files

    # Do not use logger methods for dump - go straight to console with print
    def display(self):
        rc = 0
        files = self.files()
        widths = file_stat_widths(files)
        for info in files:
            if isinstance(info, basestring):
                print(info)
            elif isinstance(info, tuple):
                itype, locn, dest = info[0:3]
                if itype in ['skip', 'dict']:
                    pass
                elif itype == 'walk':
                    print("")
                    print("%s:" % dest)
                else:
                    error("Invalid file info type '%s'" % itype)
                    rc = 1
            elif isinstance(info, list):
                print(file_stat_format(info, widths, options=self.options()))
            else:
                error("Invalid file info type %s" % type_str(info))
                rc = 1
        return rc

    def _files_to_dict(self, files):
        result = {}
        curr_locn = None
        name_index = file_stat_info_index.get('name')
        for info in files:
            if isinstance(info, basestring):
                pass
            elif isinstance(info, tuple):
                itype, locn, dest = info[0:3]
                if itype in ['skip', 'dict']:
                    pass
                elif itype == 'walk':
                    curr_locn = os.path.join(*locn)
                    result[curr_locn] = {'path': dest}
                else:
                    error("Invalid file info type '%s'" % itype)
            elif isinstance(info, list):
                if curr_locn:
                    result[curr_locn][info[name_index]] = info
            else:
                error("Invalid file info type %s" % type_str(info))
        return result

    def dict(self):
        if self._dict is not None:
            return self._dict
        self._dict = self._files_to_dict(self.files())
        return self._dict

    def _match_result(self, location, filename):
        x = json_file(filename)
        model, filevers = x and x or (None, None)
        if model: # valid conforming json config filename
            path = self.location_path(location)
            # XXX For now, if multi location, just grab the local
            # XXX We need to detect & handle no-ifs avail case (use local)
            # XXX We need to do a sync (before this)
            # XXX We need to do a compare, report mismatches
            if isinstance(path, list):
                paths = get_ifs_local_paths(path)
                if paths:
                    # Use local (non-ifs) path
                    path = paths.get('local')
                else:
                    # Errors are already reported in code
                    path = None
        else:
            path = None
        if path and isinstance(path, basestring):
            version = json_file_ver(os.path.join(path, filename))
        else:
            version = None
        blacklist = self.blacklisted(location, model, version)
        whitelist = self.whitelisted(location, model, version)
        return [model, filename, filevers, version, blacklist, whitelist]

    def _match_find(self, location, filespec):
        if not filespec:
            error("Invalid filespec: %s" % filespec)
            return []
        files = self.dict()
        if not location in files:
            error("No %s files found" % location)
            return []
        filelist = filter(lambda f: f != "path", files[location])
        if "*" in filespec or "?" in filespec:
            if filespec == "*":
                filespec = ".*"
            try:
                regex = re.compile(filespec)
            except:
                error("Invalid regular expression: %s" % filespec)
                return []
        elif JSON_DSP_FILE_REGEX.match(filespec):
            regex = re.compile("%s$" % filespec)
        elif JSON_ONEFS_FILE_REGEX.match(filespec):
            regex = re.compile("%s(\.[0-9]+)?$" % filespec)
        else:
            regex = re.compile("%s\.json(\.[0-9]+)?$" % filespec)
        result = filter(None, [regex.match(f) and f for f in filelist])
        return [self._match_result(location, r) for r in result]

    def _match_select(self, match):
        model, filename, filevers, version, blacklist, whitelist = match
        if model is None: # Not valid conforming json config filename
            return
        if version is None: # Not valid conforming json config file data
            return
        if blacklist is True:
            return # Log this info? No! - log spew!
        if whitelist is False:
            return # Log this info? No! - log spew!
        selected = self._selected
        if not model in selected:
            selected[model] = {}
        if not selected[model] or selected[model]['version'] <= version:
            selected[model].update({'filename':filename, 'version':version})

    def _match_data(self, match):
        model, filename, filevers, version, blacklist, whitelist = match
        select = self._selected.get(model, {})
        this_select = select.get('filename') == filename and "*" or ""
        return {
            'model':model, 'filename':filename, 'version':version,
            'blacklist': blacklist, 'whitelist': whitelist,
            'selected': this_select,
            }

    def match(self, location, filespec):
        # It is important to map select func over the SORTED matches,
        # in order for proper version filename selection; default sorted()
        # on matches data with ordering (model, filename, filevers) will
        # put non-versioned OneFS files *BEFORE* any versioned DSP files,
        # so that select func will always defer to DSP files over OneFS.
        self._selected = {}
        matches = self._match_find(location, filespec)
        if matches:
            self._selectState.clear_missing(location, filespec)
        else:
            # Don't repeatedly log missing filespecs over and over
            log("No %s files matching %s were found (error only logged once)" %
                (location, filespec), level=logging.ERROR,
                logfile=not self._selectState.missing(location, filespec))
            self._selectState.set_missing(location, filespec)
            return []
        # Sort first by full tuple (model,filename,filevers,version,...),
        # to get non-versioned OneFS files FIRST. Then sort again by
        # version, to handle string-based (mis)sorts for cases such as
        # [foo.1, foo.10, foo.11, foo.12, foo.13, foo.2, foo.3, etc].
        # This sequence works since Python sorts are guaranteed stable.
        matches = sorted(matches)
        matches = sorted(matches, key=lambda m:m[3]) # Sort by version
        map(self._match_select, matches)
        return map(self._match_data, matches)

    def matching(self, location, filespec):
        files = self.match(location, filespec)
        selected = filter(lambda f: f.get('selected'), files)
        blacklist = [f.get('blacklist') for f in files]
        whitelist = [f.get('whitelist') for f in files]
        headers = ["model", "filename", "version", "selected"]
        if True in blacklist:
            headers += ["blacklist"]
        if False in whitelist or True in whitelist:
            headers += ["whitelist"]
        data = [[f.get(h, '') for h in headers] for f in files]
        if data:
            headers = [h.capitalize() for h in headers]
            data_dump(data, headers, self.options())
        return files # Empty indicates no files found

    def selected(self, location, filespec):
        path = self.location_path(location)
        files = self.match(location, filespec)
        if not files:
            return None
        selected = filter(lambda f: f.get('selected'), files)
        if len(selected) == 0:
            error("No %s files matching %s were selected" %
                  (location, filespec))
            return None
        elif len(selected) > 1:
            error("Multiple (%d) %s files matching %s were selected" %
                  (len(selected), location, filespec))
            return None
        if isinstance(path, list):
            paths = get_ifs_local_paths(path)
            if not paths:
                error("Invalid location paths: %s" % path)
                return None
            path = paths['local']
        result = os.path.join(path, selected[0].get('filename'))
        if result != self._selectState.select(location, filespec):
            self._selectState.update(location, filespec, result)
            log("New %s file matching %s selected: %s" %
                (location, filespec, result), console=False)
        if self.options('fileonly'):
            return result
        if DISPLAY_SELECTED_FILE_CONTENTS:
            result = self.apply_data_overrides(location, result)
        return result


def display_files(locations, options={}):
    installed = InstalledFiles(locations, options)
    return installed.display()

# Currently, we only support "config/drives" files with
# get_matching_file and get_selected_file, but we could extend this
# in the future if necessary by manipulating location/locations.

def get_matching_files(locations, filespec, options={}):
    location = 'config/drives'
    installed = InstalledFiles(locations, options)
    files = installed.matching(location, filespec)
    return not files and 1 or 0

def get_selected_file(locations, filespec, options={}):
    location = 'config/drives'
    installed = InstalledFiles(locations, options)
    selected = installed.selected(location, filespec)
    if selected:
        if isinstance(selected, basestring):
            print(selected)
        elif isinstance(selected, list):
            for line in selected:
                print(("%s" % line).strip('\n'))
        else:
            error("Invalid selected result type: %s" % type_str(selected))
            selected = None
    return not selected and 1 or 0


##############################################################################
##  _____                        ____          _      
## | ____|_ __ _ __ ___  _ __   / ___|___   __| | ___ 
## |  _| | '__| '__/ _ \| '__| | |   / _ \ / _` |/ _ \
## | |___| |  | | | (_) | |    | |__| (_) | (_| |  __/
## |_____|_|  |_|  \___/|_|     \____\___/ \__,_|\___|
##                                                     
##############################################################################

def error_code(errorcode, options={}):
    if str(errorcode).lower() == "all":
        errors = sorted(result_strings.keys())
        width = data_width(errors)
        for errorcode in errors:
            errstr = get_result_string(errorcode)
            print("%*s: %s" % (width, errorcode, errstr))
    else:
        print(get_result_string(errorcode))
    return 0


##############################################################################
##  ____        _        _                       ____               _     
## |  _ \  __ _| |_ __ _| |__   __ _ ___  ___   / ___|_ __ ___   __| |___ 
## | | | |/ _` | __/ _` | '_ \ / _` / __|/ _ \ | |   | '_ ` _ \ / _` / __|
## | |_| | (_| | || (_| | |_) | (_| \__ \  __/ | |___| | | | | | (_| \__ \
## |____/ \__,_|\__\__,_|_.__/ \__,_|___/\___|  \____|_| |_| |_|\__,_|___/
##                                                                        
##############################################################################

def database_history(version, database_file, options={}):
    options.update({"titled":False, "time":"format"})
    options.update({"drop_columns": {"packages":[], "installs":["id", "loc"]}})
    rc = 0
    database = get_database(version, database_file, newfile=False,
                            options=options)
    if not database:
        # Errors reported in code
        rc = 1
    else:
        try:
            database.open()
            for table in ["packages", "installs"]:
                print("%s History:" % table.capitalize().rstrip('s'))
                database.dump_table_rows(table, options=options)
                print
        finally:
            database.close()
    return rc

def database_dump(version, database_file, options={}):
    rc = 0
    database = get_database(version, database_file, newfile=False,
                            options=options)
    if not database:
        # Errors reported in code
        rc = 1
    else:
        try:
            database.open()
            database.dump_tables(options)
            for table in ["packages", "locations", "installs"]:
                database.dump_table_rows(table, options=options)
        finally:
            database.close()
    return rc

def database_test(version, database_file, options={}):
    if not database_file:
        error("Invalid database_file")
        return 1
    if not 'test' in os.path.basename(database_file):
        path = os.path.dirname(database_file)
        base, ext = os.path.splitext(os.path.basename(database_file))
        database_file = os.path.join(path, "%s-test%s" % (base, ext))
        log("Using 'test' database file: %s" % database_file)
    database = UnitTestDatabase(database_file, version, newfile=True)
    return database.UnitTest()


##############################################################################
##  _____           _  __     __            _             
## |_   _|__   ___ | | \ \   / /__ _ __ ___(_) ___  _ __  
##   | |/ _ \ / _ \| |  \ \ / / _ \ '__/ __| |/ _ \| '_ \ 
##   | | (_) | (_) | |   \ V /  __/ |  \__ \ | (_) | | | |
##   |_|\___/ \___/|_|    \_/ \___|_|  |___/_|\___/|_| |_|
##                                                        
##############################################################################

def display_tool_version(options={}):
    print(get_tool_version())
    return 0


##############################################################################
##  _____           _      _        _   _                 
## |_   _|__   ___ | |    / \   ___| |_(_) ___  _ __  ___ 
##   | |/ _ \ / _ \| |   / _ \ / __| __| |/ _ \| '_ \/ __|
##   | | (_) | (_) | |  / ___ \ (__| |_| | (_) | | | \__ \
##   |_|\___/ \___/|_| /_/   \_\___|\__|_|\___/|_| |_|___/
##                                                        
##############################################################################

# If DATABASE_TRACKER==None, then "hide" the database commands,
# but the commands are still available (for Dev, Debug, etc).

_COMMANDS = [
    ('toolversion',
     { 'desc': "Display tool version",
       'func': display_tool_version,
       }),
    ('display',
     { 'desc': "Display filesystem contents",
       'func': display_files,
       'args': ['locations'],
       }),
    ('overrides',
     { 'desc': "Display override values",
       'func': overrides_display,
       'args': ['locations'],
       }),
    ('matching',
     { 'desc': "Display matching filenames",
       'func': get_matching_files,
       'args': ['locations', 'filespec'],
       'arg' : 'filespec',
       }),
    ('selected',
     { 'desc': "Display selected filename",
       'func': get_selected_file,
       'args': ['locations', 'filespec'],
       'arg' : 'filespec',
       }),
    # Database commands
    ('history',
     { 'desc': DATABASE_TRACKER and "Display database history" or None,
       'func': database_history,
       'args': ['version', 'database'],
       }),
    ('dump',
     { 'desc': DATABASE_TRACKER and "Display database contents" or None,
       'func': database_dump,
       'args': ['version', 'database'],
       }),
    ('errcode',
     { 'desc': DATABASE_TRACKER and "Display database error code" or None,
       'func': error_code,
       'arg' : 'errorcode',
       }),
    ('test',
     { 'desc': None,
       'func': database_test,
       'args': ['version', 'database'],
       }),
    # Hidden commands
    ('install',
     { 'desc': None, # "Install package",
       'func': install_files,
       'args': ['version', 'database', 'package', 'staging', 'locations'],
       }),
    ('syncifslocal',
     { 'desc': None,
       'func': sync_ifs_local,
       'args': ['locations'],
       }),
    ('help',
     { 'desc': None,
       }),
    ]

_OVERRIDES = [
    # Intentionally left overrides_display with COMMANDS
    ('blacklistadd',
     { 'desc': "Add blacklist file version",
       'func': blacklist_add,
       'args': ['locations', 'model', 'version'],
       'arg' : 'model',
       }),
    ('blacklistremove',
     { 'desc': "Remove blacklist file version",
       'func': blacklist_remove,
       'args': ['locations', 'model', 'version'],
       'arg' : 'model'
       }),
    ('blacklistremoveall',
     { 'desc': "Remove all blacklist overrides",
       'func': blacklist_delete,
       'args': ['locations', 'model'],
       'arg' : 'model'
       }),
    ('whitelistadd',
     { 'desc': "Add whitelist file version",
       'func': whitelist_add,
       'args': ['locations', 'model', 'version'],
       'arg' : 'model'
       }),
    ('whitelistremove',
     { 'desc': "Remove whitelist file version",
       'func': whitelist_remove,
       'args': ['locations', 'model', 'version'],
       'arg' : 'model'
       }),
    ('whitelistremoveall',
     { 'desc': "Remove all whitelist overrides",
       'func': whitelist_delete,
       'args': ['locations', 'model'],
       'arg' : 'model'
       }),
    ('valueadd',
     { 'desc': "Add data value override",
       'func': data_override_add,
       'args': ['locations', 'model', 'keyspec'],
       'arg2': ['delete', 'number', 'string'],
       'arg' : 'model'
       }),
    ('valueremove',
     { 'desc': "Remove data value override",
       'func': data_override_remove,
       'args': ['locations', 'model', 'keyspec'],
       'arg' : 'model'
       }),
    ('valueremoveall',
     { 'desc': "Remove all data value overrides",
       'func': data_override_delete,
       'args': ['locations', 'model'],
       'arg' : 'model'
       }),
    ]

COMMANDS = dict(_COMMANDS)
OVERRIDES = dict(_OVERRIDES)

# Override commands starting with these strings require a "version",
# which we multi-task with the normal package tool version ...
VERSION_OVERRIDES = ['blacklist', 'whitelist']

_OPTIONS = [
    #  0 == no arg -> flag
    #  1 == option with arg
    # -1 == option with no arg -> special case option
    ( 1, 'version',    "Package tool version"),
    ( 1, 'package',    "Package version"),
    ( 1, 'staging',    "Package staging directory"),
    ( 1, 'version',    "Override config file version"),
    ( 1, 'keyspec',    "Override data: Key specification"),
    ( 1, 'number',     "Override data: New numeric value"),
    ( 1, 'string',     "Override data: New string value"),
    (-1, 'delete',     "Override data: Delete given key"),
    ( 1, 'sort',       "Listing sort type"),
    ( 1, 'hash',       "Listing hash type"),
    ( 1, 'database',   "Database location"),
    ( 1, 'locations',  "Install file locations"),
    ( 0, LOCAL_FLAG,   "Local node operations only"),
    ( 0, 'debug',      "Display debug messages"),
    ( 0, 'verbose',    "Display log messages to console"),
    ( 0, 'quiet',      "Prevent log messages to console"),
    ( 0, 'fileonly',   "Display filename only (with selected)"),
    ( 0, 'framed',     "Display framed table data"),
    ( 0, 'help',       "Display this help message"),
    ]

OPTIONS = dict([(o[1], o[0]) for o in _OPTIONS])

HIDDEN_OPTIONS = [
    'sort', 'hash', 'debug', 'verbose', 'quiet', 'framed',
    'locations',
    ] + (DATABASE_TRACKER is None and ['database'] or [])

SHORT_OPTIONS = {
    'd':"debug",
    'v':"verbose",
    'q':"quiet",
    'f':"framed",
    'h':"help",
    }

# To prevent accidental typos such as "2a" from being misinterpreted
# as hex input, force requirement that hex have 0x identifier string
def hex(a):
    if str(a).lower().startswith('0x'):
        return int(a, 16)
    return int(a) # Which will raise normal ValueError if invalid

def valid_opt(a, valid):
    if a.lower() in valid:
        return a
    raise Exception

OPTION_TYPES = {
    'version': int,
    'number': [int, hex, float],
    'sort': lambda a: valid_opt(a, SORT_TYPES),
    'hash': lambda a: valid_opt(a, HASH_TYPES),
    }

##############################################################################
##  _   _                          ___      __  __       _          ____  
## | | | |___  __ _  __ _  ___    ( _ )    |  \/  | __ _(_)_ __    / /\ \ 
## | | | / __|/ _` |/ _` |/ _ \   / _ \/\  | |\/| |/ _` | | '_ \  | |  | |
## | |_| \__ \ (_| | (_| |  __/  | (_>  <  | |  | | (_| | | | | | | |  | |
##  \___/|___/\__,_|\__, |\___|   \___/\/  |_|  |_|\__,_|_|_| |_| | |  | |
##                  |___/                                          \_\/_/ 
##############################################################################

def usage(exitCode):
    vers = get_tool_version()
    ustr = 'Usage: %s COMMAND [OPTION]...' % PROGNAME
    def get_cmd(c):
        cmd, arg, desc = c[0], c[1].get('arg'), c[1].get('desc')
        return ("%s%s" % (cmd, arg and " <%s>" % arg.upper() or ''), desc)
    def get_opt(o):
        arg, opt, desc = o
        return ("--%s%s" % (opt, arg==1 and " <%s>" % opt.upper() or ''), desc)
    commands = map(get_cmd, filter(lambda a: a[1].get('desc'), _COMMANDS))
    overrides = map(get_cmd, filter(lambda a: a[1].get('desc'), _OVERRIDES))
    options = map(get_opt, filter(lambda opt: opt[1] not in HIDDEN_OPTIONS,
                                  filter(lambda o: o[0]!=0, _OPTIONS)))
    flags = map(get_opt, filter(lambda opt: opt[1] not in HIDDEN_OPTIONS,
                                filter(lambda o: o[0]==0, _OPTIONS)))
    w = data_widths(commands + overrides + options + flags)[0]
    def fmt(action, defn):
        name = action.capitalize()
        return ([name] + ["  %-*s   %s" % (w, o, d) for o, d in defn])
    cases = [(a, eval(a)) for a in
             ["commands", "overrides", "options", "flags"]]
    opts = flatten([fmt(*case) for case in cases])
    start = [vers, "", ustr, ""]
    extra = [
        "",
        "Available 'sort' types: %s" % ", ".join(SORT_TYPES),
        "Available 'hash' types: %s" % ", ".join(HASH_TYPES),
        "Default 'sort' type: %s" % DEFAULT_SORT,
        "Default 'hash' type: %s" % DEFAULT_HASH,
        "Default 'version' is current tool version: %s" % VERSION,
        "Overrides (blacklist, whitelist, value) apply to "
        "specific drive models",
        "Override changes made in %s mode are TEMPORARY," % LOCAL_FLAG,
        "and will be LOST on next ifs sync.",
        "Any filename pattern matching follows standard regex syntax.",
        ""
        ]
    # Previously sent to sys.stderr. But if we assume all failure msgs
    # are sent separately to stderr via error(), we can just use stdout
    # here -- which allows 'head' to work nicely.
    for line in start + opts + extra:
        print line
    sys.exit(exitCode)

def main(args):
    rc = 1
    logger = None
    actions = COMMANDS; actions.update(OVERRIDES)
    actions_with_opts = filter(lambda a: actions[a].get('arg'), actions)
    actions_wout_opts = filter(lambda a: a not in actions_with_opts, actions)
    actions_opts_args = filter(None, [v.get('arg') for v in actions.values()])
    get_action_opts = lambda x: remove_duplicates(flatten(
        [v.get(x, []) for v in actions.values()]))
    get_option_opts = lambda x: data_column(
        [opt[1:] for opt in filter(lambda opt: opt[0] in x, _OPTIONS)], 0)
    required, secondary = map(get_action_opts, ['args', 'arg2'])
    secondary_wout_opts = filter(lambda opt: OPTIONS[opt] in [0,-1], secondary)
    secondary_with_opts = list(set(secondary).difference(set(secondary_wout_opts)))
    optional, flags = map(get_option_opts, [[1], [0]]) # [0,-1]
    optional = list(set(optional).difference(set(required + secondary)))
    # Verify we haven't added options in a way that is confusing (non-unique)
    coptions = ['required', 'secondary', 'optional', 'flags']
    coptions = dict([(opt, eval(opt)) for opt in coptions])
    cfunc1 = lambda x, y: (
        x!=y and set(coptions[x]).intersection(set(coptions[y])) or None)
    cfunc2 = lambda c: (
        c, flatten(map( list, filter(None, [cfunc1(o, c) for o in coptions]))))
    conflicts = dict(filter(lambda c: c[1], map(cfunc2, coptions)))
    if conflicts:
        critical_error("Conflicts found in program options:")
        for c in sorted(conflicts.keys()):
            log("'%s': %s" % (c, ", ".join(conflicts[c])))
        sys.exit(1) # No usage necessary for coding errors
    sopts = ''.join(SHORT_OPTIONS.keys())
    opts = actions_with_opts + required + secondary_with_opts + optional
    lopts = ([opt + '=' for opt in opts] + actions_wout_opts + secondary_wout_opts + flags)
    action = None
    options = dict([(opt, None) for opt in opts + actions_opts_args + flags])
    options.update({
        "version": VERSION,
        "database": DATABASE_FILE,
        "locations": LOCATIONS,
        "sort": DEFAULT_SORT,
        })
    # version plays double duty: Normally this is package tool version,
    # but if override action is selected, this is the override file version
    # We allow commands to be given without '--' long opt string
    def unique_command(args):
        matches, newarg = [], None
        if args and not args[0].startswith('-'):
            arg = args[0]
            for key in actions.keys():
                if key == arg:
                    matches.append(key)
                elif key.startswith(arg):
                    matches.append(key)
            if matches:
                if len(matches) == 1:
                    newarg = matches[0]
                else:
                    check = [len(filter(lambda m: m.endswith(x), matches))
                             for x in ['remove', 'removeall']]
                    if check == [1, 1]:
                        # Find and keep the 'remove' case
                        r = filter(lambda m: m.endswith('remove'), matches)
                        newarg = r and r[0] or None
                if not newarg:
                    err = "%s is not a unique command" % arg
                    raise getopt.GetoptError(err)
        if newarg:
            args[0] = "--%s" % newarg
        return args
    def option_type_check(opt, arg):
        valid, value, types = False, None, make_list(OPTION_TYPES[opt])
        functype = type(lambda x:x)
        for typ in types:
            if type(typ) in [type, functype]:
                try:
                    value = typ(arg) ; valid = True ; break
                except:
                    valid = False
            else:
                err = "Invalid option type %s for %s" % (typ, opt)
                critical_error(err)
        return valid, value
    try:
        logger = get_logger()
        try:
            opterr = None
            opts, args = getopt.getopt(unique_command(args), sopts, lopts)
            if args:
                raise getopt.GetoptError("Invalid arguments")
            for opt, a in opts:
                o = opt.strip('-')
                if o in SHORT_OPTIONS:
                    o = SHORT_OPTIONS.get(o)
                if o in ['help']:
                    usage(0)
                elif o in actions:
                    if action is None:
                        action = o
                        arg = actions.get(action).get('arg')
                        if arg:
                            options[arg] = a
                        if filter(lambda a: action.startswith(a),
                                  VERSION_OVERRIDES):
                            # ONLY if the user has NOT already specified
                            # a version, clear out the default value
                            if not "--version" in data_column(opts, 0):
                                options['version'] = None
                    else:
                        opterr = "Multiple actions are not supported"
                elif o in OPTION_TYPES:
                    valid, value = option_type_check(o, a)
                    if valid:
                        options[o] = value
                    else:
                        opterr = "Invalid %s value: %s" % (o, a)
                elif o in required + optional:
                    options[o] = a
                elif o in secondary:
                    if o in secondary_with_opts:
                        options[o] = a
                    else:
                        options[o] = True
                elif o in flags:
                    options[o] = True
                else:
                    opterr = "Unknown argument: %s" % o
                if opterr:
                    raise getopt.GetoptError(opterr)
        except getopt.GetoptError, e:
            error('Invalid options: %s' % str(e.msg))
            usage(1)
        if not action:
            error('No action specified')
            usage(1)
        # Handle flags
        if options['verbose']:
            logger.set_verbose(True)
        if options['quiet']:
            logger.set_verbose(False)
        if options['debug']:
            logger.set_level(logging.DEBUG)
        Action = action.capitalize()
        if action in actions:
            req_args = actions[action].get('args', []) # required
            sec_args = actions[action].get('arg2', []) # secondary
            arg = actions[action].get('arg')
            if arg and not arg in req_args:
                req_args = [arg] + req_args
            if req_args:
                missing = [options.get(r) is None and r for r in req_args]
                missing = ", ".join(filter(None, missing))
                if missing:
                    error('%s: Missing options: %s' % (Action, missing))
                    usage(1)
            if sec_args:
                sdict = dict([(key, options.get(key)) for key in sec_args])
                if remove_duplicates(sdict.values()) == [None]:
                    sec_args = sorted(sec_args)
                    sec_args[-1] = "or %s" % sec_args[-1]
                    sec_args = ", ".join(sec_args)
                    error('%s: Missing options: one of %s' % (Action, sec_args))
                    usage(1)
            vals = ["%s %s" % (r, options.get(r)) for r in req_args]
            if action in ['install']:
                # Always log full install parameters
                log('%s: %s' % (Action, ', '.join(vals)))
            else:
                debug('%s: %s' % (Action, ', '.join(vals)))
            args = [options.get(a) for a in req_args] + [options]
            rc = actions[action]['func'](*args)
        else:
            error('Unknown action %s' % action)
            rc = 1
    except DatabaseError as e:
        dump_traceback("Caught Database Error: %s" % e)
        rc = 1
    except Exception as e:
        unhandled_exception()
        rc = 1
    finally:
        if logger:
            if logger.error_count():
                rc = 1
            logger.close()
    return rc

if __name__ == '__main__':
    rc = main(sys.argv[1:])
    sys.exit(rc and 1 or 0)
