#!/usr/bin/python

# Copyright C 2016 by Isilon Systems LLC-EMC-Dell.
# All rights not expressly granted herein are reserved.
# Created in the United States of America.

# TRACKING TODO LIST
#  o DONE currently console display = log file logging level, but want file >> console
#  o DONE: import of hierarchical files
#  o use of /etc/default/ files for: hwcheck replacment, ovt replacement
#  o appropriate flags to get directly at the above /etc/default files
#  o DONE: saving pass/fail results, per "test", report pass/fail status at end
#  o user friendly status reporting? for hwcheck, ovt, whatever?
#  o obviously older OVT and SKU support, P.O.C., for backfill
#  o obviously newer PSI Infinity support


import argparse
import datetime
import json
import logging
import logging.handlers as handlers
import os
import re
import sys
import time
import traceback

import isi.app.lib.procs as procs

# #############################################################################

VERSION = 1

app_name = os.path.basename(sys.argv[0])

DEFAULT_log_filename = '/var/log/%s' % app_name
DEFAULT_log_backupCount = 10
DEFAULT_log_file_level = logging.DEBUG
DEFAULT_log_disp_level = logging.INFO
DEFAULT_log_file_all_output = True

DEFAULT_input_filename = '/etc/defaults/isi_platform_check.json'

WIDTH_MIN, WIDTH_MAX = 25, 40

# #############################################################################

def init_logging():
    my_logger = {
        'disp': logging.getLogger("%s-disp" % app_name),
        'file': logging.getLogger("%s-file" % app_name),
    }
    my_logger['disp'].setLevel(DEFAULT_log_disp_level)
    my_logger['file'].setLevel(DEFAULT_log_file_level)

    if pargs.syslog:
        sh = handlers.SysLogHandler(
            address='/var/run/log',
            facility=handlers.SysLogHandler.LOG_DAEMON)
    else:
        sh = handlers.RotatingFileHandler(
            filename=pargs.logfile,
            backupCount=pargs.logcount)
        sh.doRollover() # Get a fresh new logfile
    sf = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')
    sh.setFormatter(sf)
    my_logger['file'].addHandler(sh)

    ch = logging.StreamHandler()
    cf = logging.Formatter('%(levelname)-10s: %(message)s')
    ch.setFormatter(cf)
    my_logger['disp'].addHandler(ch)

    return my_logger

def set_log_levels(levels):
    for t, lvl in levels.iteritems():
        log[t].setLevel(lvl)

def debug(level, msg, lines=True, pre=None):
    do_file = DEFAULT_log_file_all_output
    do_disp = False
    if pargs.debug >= level:
        do_file = do_disp = True
    loggers = [log[t] for t in filter(None, [do_file and 'file',
                                             do_disp and 'disp'])]
    for logger in loggers:
        if isinstance(msg, list) and lines:
            for line in msg:
                logger.debug("%s%s" % (pre and "%s: " % pre or "", line))
        else:
            logger.debug("%s%s" % (pre and "%s: " % pre or "", msg))

def verbose(level, msg, lines=True, pre=None):
    do_file = DEFAULT_log_file_all_output
    do_disp = False
    if pargs.verbose >= level:
        do_file = do_disp = True
    loggers = [log[t] for t in filter(None, [do_file and 'file',
                                             do_disp and 'disp'])]
    for logger in loggers:
        if isinstance(msg, list) and lines:
            for line in msg:
                logger.info("%s%s" % (pre and "%s: " % pre or "", line))
        else:
            logger.info("%s%s" % (pre and "%s: " % pre or "", msg))

def info(msg):
    for logger in log.itervalues():
        logger.info(msg)

def warn(msg):
    for logger in log.itervalues():
        logger.warn(msg)

def error(msg):
    for logger in log.itervalues():
        logger.error(msg)

def fail(msg):
    for logger in log.itervalues():
        logger.error(msg)

def exception(msg=''):
    log['file'].exception("EXCEPTION: %s" % msg)
    if pargs.verbose or pargs.debug:
        log['disp'].exception("EXCEPTION: %s" % msg)
    else:
        etype, evalue, etb = sys.exc_info()
        log['disp'].error("EXCEPTION: %s" % msg)
        log['disp'].error("EXCEPTION: %s, %s" % (etype, evalue))

# #############################################################################

class TestsException(Exception):
    __module__ = None

class TestsImportException(Exception):
    __module__ = None

class TestsExecException(Exception):
    __module__ = None

# #############################################################################

# So, handy ref FWIW (http://json.org/) of valid string chars:
# INVALID = [", \, any control character]
#   VALID = [ \", \\, \/, \b, \f, \n, \r, \t, \uxxxx ]
# So the JSON conf files must escape as such:
# ""This is a string"" --> "\"This is a string\""
# "echo "foo\n""       --> "echo \"foo\\n\""
# "sed 's/abc\.\(.*\)\.def.*/\1/'" --> "sed 's\/abc\\.\\(.*\\)\.def.*\/\\1\/'"
# "sed 's|abc\.\(.*\)\.def.*|\1|'" --> "sed 's|abc\\.\\(.*\\)\\.def.*|\\1|'"
# Also, for command strings that become overly long, we support command
# list-of-sub-strings, which become auto-converted into single strings.

class Tests:

    def __init__(self, filename, method=None,
                 width=80, stop_on_fail=False,
                 dump_saved_results=False,
                 include_dir='/usr/local/etc/platform'):
        self.REPORT_INCLUDE_BEFORE_RESULTS = True
        self._max_supported_version = VERSION
        self._version = None # Until determined
        self._validation = False
        self._method = method
        self._barwidth = width - 2
        self._stop_on_fail = stop_on_fail
        self._dump_saved_results = dump_saved_results
        self._include_dir = include_dir
        self._init_methods()
        self.filename = filename
        self.data = self.load(self.filename, method=method)
        self.validate(self.filename, self.data, fullvalidate=True)
        self.set_version(self.filename, self.data)
        self.saved_results_clear()

    def _init_methods(self):
        self._method_parsers = {
            # The first key element in the list of file extensions
            # is the named method to save as identifier
            ('json', ): self._parse_json,
            ('yaml', ): self._parse_yaml,
            ('toml', ): self._parse_toml,
            ('xml', ): self._parse_xml,
            ('ini', 'conf'): self._parse_ini,
            ('pkl', 'pickle'): self._parse_pickle,
        }

    def version(self):
        return self._version

    def set_version(self, filename, data):
        self._version = self.get_version(filename, data)

    def get_version(self, filename, data):
        version = data.get('meta', {}).get('version')
        if not isinstance(version, int):
            try:
                version = int(version)
            except:
                raise TestsImportException("Invalid version: %s" % version)
        if version > self._max_supported_version:
            warn("%s: File format version %s is greater than supported %s, "
                 "not all functionality may work" %
                 (filename, version, self._max_supported_version))
        return version

    # === Tests: Load and Parse ===============================================

    def load(self, filename, method=None):
        if not os.path.exists(filename):
            raise TestsImportException("File not found: %s" % filename)
        if method:
            checkmethod = str(method)
        else:
            checkmethod = os.path.splitext(filename)[1].strip('.')
        checkmethod = checkmethod.lower()
        m = p = None
        for methods, p in self._method_parsers.iteritems():
            if checkmethod in methods:
                m = methods[0]
                break
        if not m:
            raise TestsImportException("Unknown method %s" %
                                       checkmethod)
        if not p:
            raise TestsImportException("No parser available for method %s" %
                                       checkmethod)
        self.method, parser = m, p
        with open(filename) as fd:
            raw = [x.rstrip() for x in fd.readlines()]
        data = parser(raw)
        return data

    def _parse_json(self, data):
        try:
            return json.loads('\n'.join(data))
        except ValueError as e:
            raise TestsImportException(e)

    def _parse_yaml(self, data):
        # No available OneFS Python support
        raise TestsImportException("Method %s not yet implemented" % self.method)

    def _parse_toml(self, data):
        # No available OneFS Python support
        raise TestsImportException("Method %s not yet implemented" % self.method)

    def _parse_xml(self, data):
        raise TestsImportException("Method %s not yet implemented" % self.method)

    def _parse_ini(self, data):
        raise TestsImportException("Method %s not yet implemented" % self.method)

    def _parse_pickle(self, data):
        raise TestsImportException("Method %s not yet implemented" % self.method)

    # validate() only verifies the data contents of filename -- it is not
    # intended to recurse into any included files (conditional or otherwise).
    # When validate'ing the initial top-level file, includefile=False,
    # and any validation failures are treated as fatal (raise Exception).
    # Otherwise, includefile=True and they are treated as errors (return 1).
    # fullvalidate=True is used for main initialization, using dump() to walk
    # the entire available (spec'ed) config tree and performing basic validation
    # (files exist and have defined tests, etc).
    def validate(self, filename, data, includefile=False, fullvalidate=False):
        result = 0
        if includefile and fullvalidate:
            raise TestsImportException("Cannot specify both includefile "
                                       "and fullvalidate options")
        if fullvalidate:
            info("VALIDATING CONFIG INFO: %s" % filename)
        else:
            debug(1, "VALIDATING CONFIG INFO: %s" % filename)
        if not isinstance(data, dict):
            e = "File data is not a dictionary"
            if includefile:
                self.failure(e)
                return 1
            raise TestsImportException(e)
        reqd_keys = ['meta', 'tests']
        for key in reqd_keys:
            if not key in data:
                e = "Required key '%s' not in data" % key
                if includefile:
                    self.failure(e)
                    return 1
                raise TestsImportException(e)
        version = self.get_version(filename, data)
        if fullvalidate:
            try:
                self._validation = True
                result += self.dump()
            finally:
                self._validation = False
        if result:
            self.warning("VALIDATION FAILED with %s issues" % result)
        return result

    # === Tests: Execution ====================================================

    def execute(self):
        filename = self.filename
        data = self.data
        result = self.validate(filename, data)
        if result and self._stop_on_fail:
            return result
        self.print_meta(filename, data)
        self.saved_results_clear()
        self.results_clear()
        self.exec_file(filename, data)
        results = self.results()
        result = self.results_print(results)
        if self._dump_saved_results:
            self.saved_results_dump()
        return result

    def exec_file(self, filename, data):
        result = 0
        tests = self.tests(data)
        if not isinstance(tests, list):
            self.failure("%s: Invalid tests %s" % (filename, type(tests)))
            return 1
        if not tests:
            self.failure("%s: No tests defined" % (filename))
            return 1
        result = self.exec_tests(tests)
        return result

    def exec_tests(self, tests, caller_ident=None):
        result = 0
        for test in tests:
            ident = test.get('ident', caller_ident)
            testlist = test.get('testlist')
            desc = test.get('desc')
            comment = test.get('comment')
            branch = test.get('branch')
            self.print_bar()
            verbose(1, ident, pre="TESTING")
            if desc:
                verbose(1, desc, pre="DESCRIPTION")
            if not self.conditional(ident, None, test.get('conditional')):
                if branch:
                    continue
                verbose(1, "%s: conditional checks not met" %
                        ident, pre="SKIP")
                self.results_add(ident, None, {'skip': 1})
                continue
            if not testlist:
                self.failure("%s: No testlist defined" % (ident))
                result += 1
                self.results_add(ident, None, {'fail': 1})
                continue
            if comment:
                debug(1, comment, pre="COMMENT")
            result += self.exec_testlist(ident, testlist, branch)
        return result

    def exec_testlist(self, ident, testlist, branch):
        result = 0
        for test in testlist:
            result += self.exec_test(ident, test, branch)
        return result

    def exec_test(self, ident, test, branch):
        result = 0
        if not self.conditional(ident, test, test.get('conditional')):
            if branch:
                return 0
            verbose(1, "%s: conditional checks not met" %
                    self.test_name(ident, test), pre="SKIP")
            self.results_add(ident, test, {'skip': 1})
            return 0
        execs = {
            'command': self.command,
            'find-result': self.find_result,
            'tests': self.recurse,
            'include': self.include,
            'gconfig': self.gconfig,
            'gconfig-list': self.gconfig_list,
            'python': self.python,
            'placeholder': self.placeholder,
        }
        valid = filter(lambda key: key in execs, test.keys())
        if not valid:
            result = 1
            self.failure("%s: Unable to find known action in test" %
                         (self.test_name(ident, test)))
        elif len(valid) > 1:
            result = 1
            self.failure("%s: Multiple actions in test (only one allowed): %s" %
                         (self.test_name(ident, test), ", ".join(valid)))
        else:
            result = execs[valid[0]](ident, test)
        if not self.REPORT_INCLUDE_BEFORE_RESULTS or not 'include' in test:
            self.results_add(ident, test, result and {'fail': 1} or {'pass': 1})
        return result

    # === Tests: Execution: Test methods ======================================

    def command(self, ident, test):
        command = self.test_command(ident, test)
        shell = self.test_shell(ident, test)
        self.print_test_name_desc_comment(ident, test, [shell, command])
        if 'export' in test:
            exports = self.export_results(ident, test)
            if exports is None:
                return 1
            command = "%s; %s" % (exports, command)
        if shell == 0:
            return 1
        if shell:
            command = "%s -c '%s'" % (shell, command)
        # get_cmd_output uses get_popen_output, p.wait(), and osw_fix(error),
        # which "Makes compatible with os.W*", i.e.,
        # code: if error > 0: return error << 8  "Exited without a signal"
        #       else  (<= 0): return 0 - error   "Exited because of a signal"
        # os.wait():
        # Wait for completion of a child process, and return a tuple
        # containing its pid and exit status indication: a 16-bit number,
        # whose low byte is the signal number that killed the process, and
        # whose high byte is the exit status (if the signal number is zero);
        # the high bit of the low byte is set if a core file was produced.
        status, output = procs.get_cmd_output(command, include_stderr=True)
        if os.WIFEXITED(status):
            rc = os.WEXITSTATUS(status)
        else:
            warn("%s: Failed to execute %s: Stopped by signal %s" %
                 (self.test_name(ident, test), command, os.WTERMSIG(status)))
            rc = status
        rc, output = self.parse_result(ident, test, rc, output)
        verbose(3, output, pre="OUTPUT")
        result = self.check_expected(ident, test, command, rc, output)
        self.save_result(ident, test, rc, output)
        return result

    def find_result(self, ident, test):
        key = test.get('find-result')
        self.print_test_name_desc_comment(ident, test, ["Find result", key])
        rc, output = self._find_result(ident, test)
        verbose(3, output, pre="RESULT")
        result = self.check_expected(ident, test, key, rc, output)
        self.save_result(ident, test, rc, output)
        return result

    def _find_result(self, ident, spec):
        key = spec.get('find-result')
        data = self.saved_results()
        if not key in data:
            fail("Unable to find saved key '%s' in results" % key)
            return 1, []
        rc, output = 0, data[key]
        if not isinstance(output, list):
            output = [output]
        rc, output = self.parse_result(ident, spec, rc, output)
        return rc, output

    def export_results(self, ident, test):
        exports = test.get('export')
        if not isinstance(exports, list):
            exports = [exports]
        results = [self.export_result(ident, test, e) for e in exports]
        if None in results:
            return None
        return "; ".join(results)

    def export_result(self, ident, test, export):
        if not isinstance(export, dict):
            self.failure("%s: Invalid command export data" %
                         (self.test_name(ident, test)))
            return None
        key = export.get('key')
        findkey = export.get('find-result')
        if not key:
            self.failure("%s: Missing command export data key" %
                         (self.test_name(ident, test)))
            return None
        required = export.get('required', True)
        rc, output = self._find_result(ident, export)
        if rc:
            self.failure("%s: Failed to find %s results to export %s" %
                         (self.test_name(ident, test), findkey, key))
            return None
        if required and not output:
            self.failure("%s: No %s output to export for key %s" %
                         (self.test_name(ident, test), findkey, key))
            return None
        return "%s=\"%s\"" % (key, '\n'.join(map(str, output)).replace('"', "'"))

    def recurse(self, ident, test):
        tests = self.tests(test)
        if not isinstance(tests, list):
            self.failure("%s: Invalid tests %s" %
                         (self.test_name(ident, test), type(tests)))
            return 1
        if not tests:
            self.failure("%s: No tests defined" %
                         (self.test_name(ident, test)))
            return 1
        result = self.exec_tests(tests, caller_ident=test.get('name'))
        return result

    def include(self, ident, test):
        result = 0
        if self.REPORT_INCLUDE_BEFORE_RESULTS:
            self.results_add(ident, test, {})
        filename = test.get('include')
        filepath = os.path.join(self._include_dir, filename)
        self.print_test_name_desc_comment(ident, test, ["Include", filename])
        try:
            data = self.load(filepath)
            result = self.validate(filepath, data, includefile=True)
            if result:
                return result
            self.print_meta(filepath, data)
            result = self.exec_file(filepath, data)
        except TestsImportException as e:
            if self._stop_on_fail:
                raise
            exception("Exception occurred parsing %s:" % filepath)
            result = 1
        return result

    def gconfig(self, ident, test):
        gctree = test.get('gconfig-tree', "hwinventory")
        gcvar = test.get('gconfig')
        if not 'name' in test:
            test['name'] = gcvar
        command = "/usr/bin/isi_gconfig -t %s %s" % (gctree, gcvar)
        self.print_test_name_desc_comment(ident, test, command)
        status, output = procs.get_cmd_output(command, include_stderr=True)
        if os.WIFEXITED(status):
            rc = os.WEXITSTATUS(status)
        else:
            warn("%s: Failed to execute %s: Stopped by signal %s" %
                 (self.test_name(ident, test), command, os.WTERMSIG(status)))
            rc = status
        output = self.convert_gconfig(output)
        rc, output = self.parse_result(ident, test, rc, output)
        verbose(3, output, pre="OUTPUT")
        result = self.check_expected(ident, test, command, rc, output)
        self.save_result(ident, test, rc, output)
        return result

    def gconfig_list(self, ident, test):
        result = index = 0
        self.print_test_name_desc_comment(ident, test)
        gclist = test.get('gconfig-list', [])
        for gctest in gclist:
            index += 1
            gctree = gctest.get('gconfig-tree', "hwinventory")
            gcvar = gctest.get('gconfig')
            if not 'name' in gctest:
                gctest['name'] = gcvar
            gcresult = gctest.get('result')
            if None in [gcvar, gcresult]:
                result += 1
                self.failure("%s: gconfig-list entry %d invalid "
                             "(missing required test data)" %
                             (self.test_name(ident, gctest), index))
                continue
            if not 'expected' in gctest:
                gctest['expected'] = {'result': gcresult}
            command = "/usr/bin/isi_gconfig -t %s %s" % (gctree, gcvar)
            status, output = procs.get_cmd_output(command, include_stderr=True)
            if os.WIFEXITED(status):
                rc = os.WEXITSTATUS(status)
            else:
                warn("%s: Failed to execute %s: Stopped by signal %s" %
                     (self.test_name(ident, test), command, os.WTERMSIG(status)))
                rc = status
            output = self.convert_gconfig(output)
            rc, output = self.parse_result(ident, gctest, rc, output)
            verbose(3, output, pre="OUTPUT")
            result += self.check_expected(ident, gctest, command, rc, output)
            self.save_result(ident, gctest, rc, output)
        return result

    def python(self, ident, test):
        command = self.test_python(ident, test)
        shell = 'python'
        self.print_test_name_desc_comment(ident, test, [shell, command.replace('\n', '; ')])
        # NB: For this to work, you MUST escape any ' chars in script,
        #     so caveat emptor! Easier to just use escaped \" quotes,
        #     since \' is not allowed in JSON, and we would need \\'.
        command = "%s -c $'%s'" % (shell, command)
        if 'export' in test:
            exports = self.python_export_results(ident, test)
            if exports is None:
                return 1
            exports = ' '.join(exports)
            command = "%s %s" % (command, exports)
        # get_cmd_output uses get_popen_output, p.wait(), and osw_fix(error),
        # which "Makes compatible with os.W*", i.e.,
        # code: if error > 0: return error << 8  "Exited without a signal"
        #       else  (<= 0): return 0 - error   "Exited because of a signal"
        # os.wait():
        # Wait for completion of a child process, and return a tuple
        # containing its pid and exit status indication: a 16-bit number,
        # whose low byte is the signal number that killed the process, and
        # whose high byte is the exit status (if the signal number is zero);
        # the high bit of the low byte is set if a core file was produced.
        status, output = procs.get_cmd_output(command, include_stderr=True)
        if os.WIFEXITED(status):
            rc = os.WEXITSTATUS(status)
        else:
            warn("%s: Failed to execute %s: Stopped by signal %s" %
                 (self.test_name(ident, test), command, os.WTERMSIG(status)))
            rc = status
        rc, output = self.parse_result(ident, test, rc, output)
        verbose(3, output, pre="OUTPUT")
        result = self.check_expected(ident, test, command, rc, output)
        self.save_result(ident, test, rc, output)
        return result

    def python_export_results(self, ident, test):
        exports = test.get('export')
        if not isinstance(exports, list):
            exports = [exports]
        results = [self.python_export_result(ident, test, i, e)
                   for (i, e) in enumerate(exports)]
        if None in results:
            return None
        return results

    def python_export_result(self, ident, test, index, export):
        """With python exports, we use sys.argv indexing to access data.
        Therefore the normal command-style 'key' is unused for commands.
        However, for (JSON) config script and error identification,
        we do use it for messaging purposes.
        """
        if not isinstance(export, dict):
            self.failure("%s: Invalid command export data" %
                         (self.test_name(ident, test)))
            return None
        key = export.get('key', '[export entry #%s]' % index)
        findkey = export.get('find-result')
        required = export.get('required', True)
        rc, output = self._find_result(ident, export)
        if rc:
            self.failure("%s: Failed to find %s results to export %s" %
                         (self.test_name(ident, test), findkey, key))
            return None
        if required and not output:
            self.failure("%s: No %s output to export for key %s" %
                         (self.test_name(ident, test), findkey, key))
            return None
        escape = lambda s: str(s).replace('"', '\\"')
        output = map(escape, output)
        if len(output) == 0:
            return None
        elif len(output) == 1:
            return "\"%s\"" % output[0]
        else:
            return "\"[%s]\"" % (' @@ '.join(output))

    def placeholder(self, ident, test):
        placeholder = test.get('placeholder')
        self.print_test_name_desc_comment(ident, test, 'placeholder')
        if not isinstance(placeholder, bool) or placeholder != True:
            self.failure("%s: Invalid placeholder: Expected boolean 'True'" %
                         (self.test_name(ident, test)))
            return 1
        return 0

    # === Tests: Data Manipulation (parse, regex, replace, etc) ===============

    def parse_result(self, ident, test, rc, output):
        tname = self.test_name(ident, test)
        parse = test.get('parse-result')
        if not parse:
            return rc, output
        known_parsers = ['regex', 'replace', 'sub', 'drive-excluded-bays']
        valid = filter(lambda key: key in known_parsers, parse.keys())
        if not valid:
            self.failure("%s: Invalid parser data: No valid parsers defined" % tname)
            rc = 1
        invalid = filter(lambda key: key not in known_parsers, parse.keys())
        if invalid:
            self.failure("%s: Invalid parser data: Unknown parsers: %s" %
                         (tname, ", ".join(invalid)))
            rc = 1
        regex = parse.get('regex')
        replace = parse.get('replace')
        sub = parse.get('sub')
        data = output
        if 'drive-excluded-bays' in parse:
            data = self.exclude_drive_bays(ident, test, data)
        if regex:
            data = self.regex(ident, test, data, regex)
        if replace:
            data = self.replace(ident, test, data, replace)
        if sub:
            data = self.sub(ident, test, data, sub)
        if not data:
            rc = 1
        return rc, data

    def regex(self, ident, test, data, regex):
        d = data
        if isinstance(regex, basestring):
            regex = [regex]
        elif isinstance(regex, tuple):
            regex = list(regex)
        elif not isinstance(regex, list):
            self.failure(
                "%s: Invalid parse-result regex %s" %
                (self.test_name(ident, test), type(regex)))
            return d
        for expr in regex:
            r = re.compile(expr)
            d = filter(lambda x: r.match(x), d)
        debug(3, "Applying regex filters reduced %s lines to %s" %
              (len(data), len(d)))
        return d

    def replace(self, ident, test, data, replace):
        d = data
        if isinstance(replace, basestring):
            replace = [replace]
        elif isinstance(replace, tuple):
            replace = list(replace)
        elif not isinstance(replace, list):
            self.failure(
                "%s: Invalid parse-result replace %s" %
                (self.test_name(ident, test), type(replace)))
            return d
        repl = [x.split(',') for x in replace]
        if set(map(len, repl)) != set([2]):
            self.failure(
                "%s: Invalid parse-result regex values: %s "
                "(must be CSV 'match,replace' strings)" %
                (self.test_name(ident, test), x))
            return d
        for old, new in repl:
            d = [x.replace(old, new) for x in d]
        return d

    def sub(self, ident, test, data, sub):
        d = data
        if not isinstance(sub, list):
            self.failure(
                "%s: Invalid parse-result sub '%s', %s" %
                (self.test_name(ident, test), sub, type(sub)))
            return d
        if ((not len(sub) == 2) or
            (not isinstance(sub[0], basestring)) or
            (not isinstance(sub[1], basestring))):
            self.failure(
                "%s: Invalid parse-result sub %s" %
                (self.test_name(ident, test), sub))
            return d
        patt, repl = sub
        d = [re.sub(patt, repl, x) for x in data]
        return d

    def convert_gconfig(self, output):
        result = []
        r = re.compile("^([^ ]*) \(([^)]*)\) = (.*)$")
        for o in output:
            m = r.match(o)
            if m:
                k, t, v = m.groups()
                if t == 'bool' and v == 'false':
                    result.append(False)
                elif t == 'bool' and v == 'true':
                    result.append(True)
                elif t == 'bool':
                    result.append(v)
                elif t == 'double':
                    result.append(float(v))
                elif t.startswith('int'):
                    result.append(int(v))
                elif t.startswith('uint'):
                    result.append(int(v))
                else: # including char*
                    result.append(v)
            else:
                result.append(o)
        return result

    def exclude_drive_bays(self, ident, test, data):
        # Expect output = single line string w/list of bay numbers,
        # possibly with special case "1/2" for the either-or case.
        # For simplicity, just hard-code handling "1/2" (explicit).
        exlude = []
        search = {
            'name': 'drive-excluded-bays',
            'find-result': 'drive-excluded-bays'
        }
        rc, output = self._find_result(ident, search)
        if rc or not output:
            # Do nothing (no parsing)
            return data
        s = str(output[0])
        regexs = [
            "^\[([0-9]+/[0-9]+)\]$",
            "^\[([0-9, ]+)\]$",
            "^\[([, ]*)\]$",
        ]
        m = [re.match(r, s) for r in regexs]
        if m[0]:
            e = m[0].group(1).strip()
            if e != "1/2":
                self.failure("%s: No support for exlude drive bays %s (yet)" %
                             (self.test_name(ident, test), s))
            else:
                r = re.compile("Bay *[12][: ]")
                temp = []
                for d in data:
                    keep = True
                    if r.match(d):
                        # Either-or: Use "Unknown" presence as missing indicator
                        for x in ["not a recognized", "Tech Unknown Unknown", "UNKNOWN"]:
                            if x in d:
                                keep = False
                                break
                    if keep:
                        temp.append(d)
                data = temp
        elif m[1]:
            try:
                e = map(int, filter(None, [o.strip() for o in m[1].group(1).split(',')]))
            except ValueError:
                self.failure("%s: Invalid exclude drive bays data %s:"
                             "Unable to convert to integer bays" %
                             (self.test_name(ident, test), s))
                e = []
            for bay in e:
                r = re.compile("Bay *%d[: ]" % bay)
                temp = []
                for d in data:
                    if not r.match(d):
                        temp.append(d)
                data = temp
        elif m[2]:
            # empty list
            pass
        else:
            # Oops, no match?
            self.failure("%s: Invalid exclude drive bays data %s" %
                         (self.test_name(ident, test), s))
        return data

    # === Tests: Checks: Expected Result Verification =========================

    def check_expected(self, ident, test, command, rc, output):
        result = 0
        tname = self.test_name(ident, test)
        if not 'expected' in test:
            return 0
        expected = test.get('expected')
        if not isinstance(expected, dict):
            self.failure("%s: Invalid expected data: Not dictionary format" % tname)
            return 1
        # Check at at least one of our expected expecteds are present
        expected_checks = ['exitcode', 'result', 'regex', 'hwmon-log']
        valid_checks = filter(lambda key: key in expected_checks, expected.keys())
        if not valid_checks:
            self.failure("%s: Invalid expected data: No valid checks defined" % tname)
            return 1
        invalid_checks = filter(lambda key: key not in expected_checks, expected.keys())
        if invalid_checks:
            self.failure("%s: Invalid expected data: Unknown invalid checks: %s" %
                         (tname, ", ".join(invalid_checks)))
            return 1
        # expected result||regex entries can be simple strings, or more
        # complex dicts with {string, bools (all, required)}.
        # Simple string == {string, all=false, required=true}.
        # exitcode entries are trivial, effectively {all=false, required=true}.
        # If all=true, every line in output must match; misses = failures.
        # If required=true, at least one line in output must match.
        if 'exitcode' in expected:
            expds = expected['exitcode']
            errors = self.check_exitcode(rc, expds)
            debug(1, "rc=%s expds=%s errors=%s" % (rc, expds, errors),
                  pre="CHECK EXPECTED EXITCODE")
            if errors:
                result = 1
                for error in errors:
                    fail("%s: %s" % (tname, error))
            else:
                verbose(2, "%s: exitcode: %s" % (tname, rc), pre="PASS")
        if 'result' in expected:
            expds = expected['result']
            errors = self.check_result(output, expds)
            debug(1, "output=%s expds=%s errors=%s" % (output, expds, errors),
                  pre="CHECK EXPECTED RESULT")
            if errors:
                result = 1
                for error in errors:
                    fail("%s: %s" % (tname, error))
            else:
                # NB: To limit logging, we ONLY display FIRST line of output
                verbose(2, "%s: result: %s" %
                        (tname, output and output[0]), pre="PASS")
        if 'regex' in expected:
            expds = expected['regex']
            errors = self.check_regex(output, expds)
            debug(1, "output=%s expds=%s errors=%s" % (output, expds, errors),
                  pre="CHECK EXPECTED RESULT")
            if errors:
                result = 1
                for error in errors:
                    fail("%s: %s" % (tname, error))
            else:
                # NB: To limit logging, we ONLY display FIRST line of output
                verbose(2, "%s: result: %s" %
                        (tname, output and output[0]), pre="PASS")
        if 'hwmon-log' in expected:
            errors = self.check_hwmon_log(output)
            debug(1, "output=%s hwmon=True errors=%s" % (output, errors),
                  pre="CHECK EXPECTED HWMON")
            if errors:
                result = 1
                for error in errors:
                    fail("%s: %s" % (tname, error))
            else:
                verbose(2, "%s: result: HWMON History Log Check Successful" %
                        (tname), pre="PASS")
        return result

    def check_exitcode(self, rc, expd):
        errors = []
        if not isinstance(rc, int):
            errors.append("Invalid exit code %s %s (expecting int)" %
                          (rc, type(rc)))
        elif isinstance(expd, list):
            invalids = filter(None, map(lambda x: not isinstance(x, int), expd))
            if invalids:
                errors.append("Invalid expected exit codes %s %s "
                              "(expecting int or list of ints)" %
                              (expd, type(expd)))
            else:
                valids = filter(None, map(lambda x: rc == x, expd))
                if not valids:
                    errors.append("exit code %s != expected %s" % (rc, expd))
        elif not isinstance(expd, int):
            errors.append("Invalid expected exit code %s %s (expecting int)" %
                          (expd, type(expd)))
        elif rc != expd:
            errors.append("exit code %s != expected %s" % (rc, expd))
        return errors

    def check_result(self, output, expd):
        errors = []
        #if not output:
        #    errors.append("No output to compare with expected result %s" % expd)
        #el
        if not isinstance(output, list):
            errors.append("Invalid output (expecting list of results)")
        elif isinstance(expd, (basestring, bool, int, float, list)):
            errors.extend(self.check_result_dict(
                output, {'result': expd, 'all': False, 'required': True}))
        elif isinstance(expd, dict):
            if not 'result' in expd and not 'range2' in expd:
                errors.append("Missing result string in expected result dict spec")
            else:
                errors.extend(self.check_result_dict(output, expd))
        else:
            errors.append("Invalid expected result %s" % expd)
        return errors

    def check_result_dict(self, output, expd_dict):
        errors = []
        known_checks = ['result', 'range', 'range2']
        valid_checks = filter(lambda key: key in known_checks, expd_dict.keys())
        if not valid_checks:
            errors.append(
                "Invalid expected data: No valid result checks defined")
        known_checks += ['all', 'required']
        invalid_checks = filter(lambda key: key not in known_checks, expd_dict.keys())
        if invalid_checks:
            errors.append(
                "Invalid expected data: Unknown invalid result checks: %s" %
                ", ".join(invalid_checks))
        if errors:
            return errors
        if 'range' in expd_dict:
            return self.check_result_range(output, expd_dict)
        if 'range2' in expd_dict:
            return self.check_result_range2(output, expd_dict)
        expd_results = expd_dict.get('result')
        if not isinstance(expd_results, list):
            expd_results = [expd_results]
        all_reqd = expd_dict.get('all')
        required = expd_dict.get('required')
        matches, misses = [], []
        for o in output:
            match = False
            for expd_result in expd_results:
                if isinstance(expd_result, basestring):
                    op = str
                elif isinstance(expd_result, bool):
                    op = bool
                elif isinstance(expd_result, int):
                    op = int
                elif isinstance(expd_result, float):
                    op = float
                else:
                    op = lambda x: x
                try:
                    opresult = op(o)
                except ValueError:
                    self.failure("Unable to convert %s(%s)" % (op, o))
                    opresult = None
                if expd_result == opresult:
                    match = True
                    break
            if match:
                matches.append(o)
            else:
                misses.append(o)
        cmatch, cm, co = [len(filter(lambda x: x is not None, x))
                          for x in (matches, misses, output)]
        if all_reqd and cm:
            errors.append("%d of %d output lines did NOT match result %s" %
                          (cm, co, expd_dict))
            index = 0
            for line, miss in enumerate(misses, 1):
                if miss is not None:
                    index += 1
                    errors.append("Result miss %d of %d: Line %d: %s" %
                                  (index, cm, line, miss))
        if required and not cmatch:
            errors.append("0 of %d output lines matched result %s" %
                          (co, expd_dict))
        return errors

    def check_result_range(self, output, expd_dict):
        errors = []
        expd_value = float(expd_dict.get('result'))
        expd_range = expd_dict.get('range')
        if not isinstance(expd_range, float):
            errors.append("Invalid required result range type %s" %
                          (type(expd_range)))
            return errors
        mn, mnp = expd_value * (1 - expd_range), (1 - expd_range)
        mx, mxp = expd_value * (1 + expd_range), (1 + expd_range)
        all_reqd = expd_dict.get('all')
        required = expd_dict.get('required')
        values, matches, misses = [], [], []
        for o in output:
            try:
                values.append(float(o))
            except ValueError:
                errors.append("Invalid result %s: Cannot perform numeric "
                              "result range check" % o)
        for v in values:
            if v < mn:
                errors.append("Range check failed: %.0f < %.2f (%.4f%% %.0f)" %
                              (v, mn, 100*mnp, expd_value))
                misses.append(v)
            elif v > mx:
                errors.append("Range check failed: %.0f > %.2f (%.4f%% %.0f)" %
                              (v, mx, 100*mxp, expd_value))
                misses.append(v)
            else:
                matches.append(v)
        if all_reqd and misses:
            errors.append("%d of %d output lines did NOT match result %s" %
                          (len(misses), len(output), expd_dict))
        if required and not matches:
            errors.append("0 of %d output lines matched result %s" %
                          (len(output), expd_dict))
        return errors

    def check_result_range2(self, output, expd_dict):
        """A simpler (?) range check format: CSV results expected.
        Either "NNN, NNN" (with arbitrary whitespace), or optionally they
        can be named as "name1:NNN, name2:NNN" or "name1=NNN, name2=NNN".
        For simplicity, if named, they must both be named."""
        errors = []
        expd_range = expd_dict.get('range2')
        if not isinstance(expd_range, float):
            errors.append("Invalid required result range type %s" %
                          (type(expd_range)))
            return errors
        minp = (1 - expd_range)
        maxp = (1 + expd_range)
        dec = len(str(int(1/expd_range)))
        all_reqd = expd_dict.get('all')
        required = expd_dict.get('required')
        values, matches, misses = [], [], []
        for o in output:
            n = [n.strip() for n in o.split(',')]
            if len(n) != 2:
                errors.append("Invalid result %s: "
                              "'range2' requires 2 numerical CSV values" % o)
            else:
                v = None
                try:
                    match = [re.match('(.+)\s*[:=]\s*([0-9.]+)', x) for x in n]
                    if not None in match:
                        v = [(g.group(1), float(g.group(2))) for g in match]
                    else:
                        match = [re.match('([0-9.]+)', x) for x in n]
                        if not None in match:
                            v = [('value1', float(match[0].group(1))),
                                 ('value2', float(match[1].group(1)))]
                    if v:
                        values.append(v)
                    else:
                        errors.append("Invalid result %s: "
                                      "Unable to identify 2 numerical CSV values" % o)
                except ValueError:
                    errors.append("Invalid result %s: Cannot perform numeric "
                                  "result range check" % o)
        for v in values:
            (n1, v1), (n2, v2) = v
            if v1 == v2:
                # handles v1==v2==0
                ratio = 1.0
            elif v2 != 0:
                ratio = v1 / v2
            else:
                temp = n1, v1
                n1, v1 = n2, v2
                n2, v2 = temp
                ratio = v1 / v2
            if v1 == v2:
                # Just in case.. explicitly match equality!
                matches.append(v)
            if ratio < minp:
                errors.append(
                    "Range check failed: %s=%.0f / %s=%.0f = %.*f < %.*f (%.*f%%)" %
                    (n1, v1, n2, v2, dec, ratio, dec, minp, dec, 100 * expd_range))
                misses.append(v)
            elif ratio > maxp:
                errors.append(
                    "Range check failed: %s=%.0f / %s=%.0f = %.*f > %.*f (%.*f%%)" %
                    (n1, v1, n2, v2, dec, ratio, dec, maxp, dec, 100 * expd_range))
                misses.append(v)
            else:
                matches.append(v)
        if all_reqd and misses:
            errors.append("%d of %d output lines did NOT match result %s" %
                          (len(misses), len(output), expd_dict))
        if required and not matches:
            errors.append("0 of %d output lines matched result %s" %
                          (len(output), expd_dict))
        return errors

    def check_regex(self, output, expd):
        errors = []
        #if not output and required:
        #    errors.append("No output to compare with expected regex %s" % expd)
        #elif not output:
        #    output = ['']
        if not output:
            output = ['']
        #elif not isinstance(output, list):
        if not isinstance(output, list):
            errors.append("Invalid output (expecting list of results)")
        elif isinstance(expd, (basestring, list)):
            errors.extend(self.check_regex_dict(
                output, {'regex': expd, 'all': False, 'required': True}))
        elif isinstance(expd, dict):
            if not 'regex' in expd:
                errors.append("Missing regex string in expected regex dict spec")
            elif not isinstance(expd.get('regex'), basestring):
                errors.append("Invalid expected regex %s" % expd)
            else:
                errors.extend(self.check_regex_dict(output, expd))
        else:
            errors.append("Invalid expected regex %s" % expd)
        return errors

    def check_regex_dict(self, output, expd_dict):
        errors = []
        expd_regex = expd_dict.get('regex')
        if not isinstance(expd_regex, list):
            expd_regex = [expd_regex]
        all_reqd = expd_dict.get('all')
        required = expd_dict.get('required')
        matches, misses = [], []
        regex = [re.compile(r) for r in expd_regex]
        for o in output:
            match = None
            for r in regex:
                match = r.match(o)
                if match:
                    break
            if match:
                matches.append(match.string)
            else:
                misses.append(o)
        cmatch, cm, co = [len(filter(None, x)) for x in (matches, misses, output)]
        if all_reqd and filter(None, misses):
            errors.append("%d of %d output lines did NOT match regex %s" %
                          (cm, co, expd_dict))
            index = 0
            for line, miss in enumerate(misses, 1):
                if miss:
                    index += 1
                    errors.append("Regex miss %d of %d: Line %d: %s" %
                                  (index, cm, line, miss))
        if required and not filter(None, matches):
            errors.append("0 of %d output lines matched regex %s" %
                          (co, expd_dict))
        return errors

    def check_hwmon_log(self, output):
        def dump_hwmon_log():
            wcount = len(str(len(output)))
            return ["%0*d: %s" % (wcount, line, out)
                    for line, out in enumerate(output, 1)]
        errors = []
        ehdr = "Invalid JSON data in HWMON History Log data:"
        try:
            data = json.loads('\n'.join(output))
        except ValueError as e:
            errors.append(ehdr)
            errors.append(str(e))
            errors.extend(dump_hwmon_log())
            return errors
        # For now simple success criteria:
        # Expect HMON History Log JSON data to all be:
        # key : {empty dict}
        # and invalid (errored) non-empty dicts to all be:
        # { identkey: errorstring }
        if not isinstance(data, dict):
            errors.append(ehdr)
            errors.append("Expected dictionary result not %s" % type(data))
            errors.extend(dump_hwmon_log())
            return errors
        for key in sorted(data.keys()):
            result = data[key]
            if not isinstance(result, dict):
                # For cleanliness, convert unicode to simple string
                if isinstance(result, unicode):
                    result = str(result)
                errors.append(
                    "Invalid '%s' result: expected dict not %s" %
                    (key, type(result)))
                continue
            if not result.keys():
                # Success
                continue
            for failkey in sorted(result.keys()):
                failresult = result[failkey]
                if not isinstance(failresult, basestring):
                    errors.append(
                        "Invalid '%s:%s' result: expected string not %s" %
                        (key, failkey, type(failresult)))
                    errors.append(
                        "%s.%s data: %s" % (key, failkey, failresult))
                    continue
                errors.append("%s:%s: %s" % (key, failkey, failresult))
        return errors

    # === Tests: Results ======================================================

    def results_clear(self):
        self._results = []

    def results(self):
        return self._results

    def results_add(self, ident, test={}, counts={}):
        if not test:
            test = {}
        self._results.append([(ident, test.get('name')), counts])

    def results_print(self, results, sep=' ', align=True):
        # align: Do we want aligned keys? Or all flush left?
        reqd_keys = ['pass', 'fail', 'skip']
        counts = dict([(k, sum([r[1].get(k, 0) for r in results])) for k in reqd_keys])
        if self._validation:
            return counts['fail']
        total  = sum(counts.values())
        sident = map(str, [r[0][0] for r in results])
        stest  = map(str, [r[0][1] for r in results])
        wident = self.min_max_bound(WIDTH_MIN, WIDTH_MAX, map(len, sident))
        wtest  = self.min_max_bound(WIDTH_MIN, WIDTH_MAX, map(len, stest))
        reqd_w = dict([(k, len(str(counts[k]))) for k in reqd_keys])
        def rstr(rcounts, fcounts=True):
            # fcounts=True: Filter out all zero counts
            keys = sorted(rcounts.keys())
            [key in keys and keys.remove(key) for key in reqd_keys]
            keys = reqd_keys + keys
            s = []
            for k in keys:
                c = rcounts.get(k, 0)
                if c or not fcounts:
                    s.append('%s:%-*s' % (k, reqd_w.get(k, 1), c))
                elif align:
                    s.append('%*s' % (len(k) + 1 + reqd_w.get(k, 1), ''))
            return sep.join(s)
        sstr = rstr(counts, fcounts=False)
        hstr = ("%-*s:%-*s:%s" %
                (wident, "-" * (wident), wtest, "-" * (wtest+1), "-" * (len(sstr)+1)))
        self.print_bar(width=len(hstr))
        info(hstr)
        info("%-*s: %-*s: %s" %
             (wident, "SUMMARY", wtest, "%s TOTAL TESTS" % total, sstr))
        info(hstr)
        for r in results:
            ident = r[0][0]
            test  = r[0][1] or '' # Convert None into empty string
            rs = rstr(r[1], fcounts=True)
            info("%-*s: %-*s: %s" % (wident, ident, wtest, test, rs))
        self.print_bar(width=len(hstr))
        return counts['fail']

    # === Tests: Saved Results ================================================

    def saved_results_clear(self):
        self._saved_results = {}

    def saved_results(self):
        return self._saved_results

    def save_result(self, ident, test, rc, output):
        if not 'save-result' in test:
            return
        save = test.get('save-result')
        if not isinstance(save, dict):
            self.failure(
                "%s: Invalid save-result %s" %
                (self.test_name(ident, test), type(save)))
            return
        if not 'key' in save:
            self.failure(
                "%s: Missing 'key' for save-result" %
                (self.test_name(ident, test)))
            return
        key = save['key']
        save_exitcode = save.get('exitcode')
        save_successonly = save.get('success only')
        save_default = save.get('default')
        save_list = save.get('list')
        if save_list and (save_exitcode or save_successonly or save_default):
            self.failure(
                "%s: Invalid inconsistent settings for save-result: "
                "save list=%s, but also have one of %s set" %
                (self.test_name(ident, test), save_list,
                 ','.join(['exitcode', 'success only', 'default'])))
            return
        if save_exitcode and (save_list or save_successonly or save_default):
            self.failure(
                "%s: Invalid inconsistent settings for save-result: "
                "save exitcode=%s, but also have one of %s set" %
                (self.test_name(ident, test), save_list,
                 ','.join(['list', 'success only', 'default'])))
            return
        val = None
        if save_exitcode:
            val = rc
        elif save_successonly and not rc == 0:
            if save_default:
                val = save_default
        elif save_list:
            val = output
        else:
            val = output and output[0]
        if not self._validation:
            if save_successonly and val is None:
                debug(2, "NOT SAVING [%s], since success only = %s" %
                      (key, save_successonly))
            else:
                x = 1000
                debug(2, "SAVING [%s]=%s%s" %
                      (key, str(val)[:x], len(str(val)) > x and "[...]" or ""))
                self._saved_results[key] = val

    def saved_results_dump(self):
        data = self.saved_results()
        keys = sorted(data.keys())
        width = self.min_max_bound(WIDTH_MIN, WIDTH_MAX, map(len, keys))
        self.print_bar()
        info("%-*s: %s" % (width, "-" * width, "-" * (self._barwidth - width - 2)))
        info("%-*s: %s" % (width, "SAVED KEYS", "%s total" % len(keys)))
        info("%-*s: %s" % (width, "-" * width, "-" * (self._barwidth - width - 2)))
        for key in keys:
            val = data[key]
            if isinstance(val, list):
                for v in val:
                    info("%-*s: %s" % (width, key, v))
            else:
                info("%-*s: %s" % (width, key, val))

    # === Tests: Conditionals =================================================

    def conditional(self, ident, test, conditionals):
        if not conditionals:
            return True
        if isinstance(conditionals, dict):
            conditionals = [conditionals]
        checks = [self.cond_check(ident, test, cond) for cond in conditionals]
        debug(2, "%s: Conditional Result: %s" %
              (self.test_name(ident, test), checks))
        return reduce(lambda a, b: a and b, checks)

    def cond_check(self, ident, test, cond):
        key = cond.get('key')
        data = self.saved_results()
        debug(3, "%s: Conditional Check: %s, data[%s]=%s %s" %
              (self.test_name(ident, test), cond,
               key, data.get(key), type(data.get(key))))
        if key in data:
            if 'value' in cond:
                if str(cond.get('value')) == str(data.get(key)):
                    return True
                return False
            if 'regex' in cond:
                if re.match(cond.get('regex'), str(data.get(key))):
                    return True
                return False
            # Else key in data, but no value or regex - missing SOMETHING, so...
            name = test and test.get('name')
            warn("%s: Missing 'value' or 'regex' in conditional check for '%s'" %
                 (self.test_name(ident, test), key))
            return False
        # Else no key in data; no other conditional support at this time
        return True

    # === Tests: Helper Utilities =============================================

    def tests(self, data=None):
        if data is None:
            data = self.data
        return data.get('tests')

    def test_command(self, ident, test):
        # As a convenience, since JSON doesn't trivially allow easy
        # multiline strings, we allow JSON command lists to be
        # auto-converted into single strings to allow the config
        # files to be easier for humans to parse
        command = test.get('command')
        if isinstance(command, list):
            if filter(lambda c: not isinstance(c, basestring), command):
                raise TestsImportException(
                    "%s: Invalid command: List of non-strings" %
                    (self.test_name(ident, test)))
            command = ' '.join(command)
        return command

    def test_shell(self, ident, test):
        shell = test.get('shell')
        if not shell:
            return None
        if not isinstance(shell, basestring):
            self.failure("%s: Invalid command shell '%s'" %
                         (self.test_name(ident, test), shell))
            return 0
        if not '/' in shell:
            shell = os.path.join('/bin', shell)
        if not os.path.exists(shell):
            self.failure("%s: Invalid command shell '%s': does not exist" %
                         (self.test_name(ident, test), shell))
            return 0
        if not os.access(shell, os.X_OK):
            self.failure("%s: Invalid command shell '%s': not executable" %
                         (self.test_name(ident, test), shell))
            return 0
        return shell

    def test_python(self, ident, test):
        # See test_command()
        python = test.get('python')
        if isinstance(python, list):
            if filter(lambda c: not isinstance(c, basestring), python):
                raise TestsImportException(
                    "%s: Invalid python: List of non-strings" %
                    (self.test_name(ident, test)))
            python = '\n'.join(python)
        return python

    def test_name(self, ident, test, extra=[]):
        name = test and test.get('name')
        if not isinstance(extra, list):
            extra = [extra]
        extra = filter(None, extra)
        return ": ".join(filter(None, [ident, name] + extra))

    def test_desc(self, ident, test):
        name = self.test_name(ident, test)
        desc = test and test.get('desc') or []
        if not isinstance(desc, list):
            desc = [desc]
        return ["%s: %s" % (name, d) for d in desc]

    def print_test_name_desc_comment(self, ident, test, extra=[]):
        desc = self.test_desc(ident, test)
        name = self.test_name(ident, test, extra)
        for d in desc:
            verbose(2, d, pre="DESC")
        verbose(2, name, pre="TEST")
        comment = test.get('comment')
        if comment:
            debug(1, comment, pre="COMMENT")

    def meta(self, data=None):
        if data is None:
            data = self.data
        return data.get('meta')

    def print_meta(self, filename=None, data=None):
        if filename is None:
            filename = self.filename
        if data is None:
            data = self.data
        meta = self.meta(data)
        self.print_bar()
        # debug(1, "META: %s" % meta)
        pre = "CONFIG FILE"
        info("%s: Path: %s" % (pre, filename))
        info("%s: Name: %s" % (pre, meta.get('name')))
        info("%s: Date: %s" % (pre, meta.get('date')))
        info("%s: Version: %s" % (pre, meta.get('version')))
        description = meta.get('description')
        if not isinstance(description, list):
            info("%s: Description: %s" % (pre, description))
        else:
            info("%s: Description:" % (pre))
            for line in description:
                info("%s: - %s" % (pre, line))

    def print_bar(self, width=None):
        if width is None:
            width = self._barwidth
        verbose(1, "=" * width)

    def failure(self, msg):
        if self._stop_on_fail:
            raise TestsExecException(msg)
        fail(msg)

    def warning(self, msg):
        warn(msg)

    def min_max_bound(self, vmin, vmax, vals):
        if not isinstance(vals, list):
            vals = [vals]
        return min(vmax, max(vmin, max(vals)))

    # === Tests: Dump Data ====================================================

    # NOTE: If self._validation is True (see: validate()), then calling dump()
    # will not display any of the normal info() dump output, but will walk
    # the same code path tracking errors for validate().

    def dump(self):
        # C.f.: execute()
        filename = self.filename
        data = self.data
        if not self._validation:
            info("DUMP: %s" % filename)
            self.print_meta(filename, data)
        self.saved_results_clear()
        self.results_clear()
        self.dump_file(filename, data)
        results = self.results()
        result = self.results_print(results)
        return result

    def dump_file(self, filename, data):
        # C.f.: exec_file()
        result = 0
        tests = self.tests(data)
        if not isinstance(tests, list):
            self.warning("%s: Invalid tests %s" % (filename, type(tests)))
            return 1
        if not tests:
            self.warning("%s: No tests defined" % (filename))
            return 1
        result = self.dump_tests(tests)
        return result

    def dump_tests(self, tests, caller_ident=None):
        # C.f.: exec_tests()
        result = 0
        for test in tests:
            ident = test.get('ident', caller_ident)
            testlist = test.get('testlist')
            desc = test.get('desc')
            branch = test.get('branch')
            if not self._validation:
                self.print_bar()
                info("DUMP: %s" % ident)
                if desc:
                    info("DESC: %s" % desc)
            self.conditional_dump(ident, None, test.get('conditional'))
            if not testlist:
                self.warning("%s: No testlist defined" % (ident))
                result += 1
                self.results_add(ident, None, {'fail': 1})
                continue
            result += self.dump_testlist(ident, testlist, branch)
        return result

    def dump_testlist(self, ident, testlist, branch):
        # C.f.: exec_testlist()
        result = 0
        for test in testlist:
            result += self.dump_test(ident, test, branch)
        return result

    def dump_test(self, ident, test, branch):
        # C.f.: ex_test()
        result = 0
        self.conditional_dump(ident, test, test.get('conditional'))
        if 'command' in test:
            result = self.dump_command(ident, test)
        elif 'find-result' in test:
            result = self.dump_find_result(ident, test)
        elif 'tests' in test:
            tests = self.tests(test)
            if not isinstance(tests, list):
                result = 1
                self.warning("%s: Invalid tests %s" %
                             (self.test_name(ident, test), type(tests)))
            elif not tests:
                result = 1
                self.warning("%s: No tests defined" %
                             (self.test_name(ident, test)))
            else:
                result = self.dump_tests(tests, caller_ident=test.get('name'))
        elif 'include' in test:
            result = self.dump_include(ident, test)
        elif 'gconfig' in test:
            result = self.dump_gconfig(ident, test)
        elif 'gconfig-list' in test:
            result = self.dump_gconfig_list(ident, test)
        elif 'python' in test:
            result = self.dump_python(ident, test)
        elif 'placeholder' in test:
            result = self.dump_placeholder(ident, test)
        else:
            result = 1
            self.warning("%s: Unable to find known action in test" %
                         (self.test_name(ident, test)))
        self.results_add(ident, test, result and {'fail': 1} or {'pass': 1})
        return result

    def dump_command(self, ident, test):
        # C.f.: command()
        command = self.test_command(ident, test)
        if not self._validation:
            info("DUMP: %s" % self.test_name(ident, test, command))
        result, rc, output = 0, 0, ['dump']
        self.save_result(ident, test, rc, output)
        return result

    def dump_find_result(self, ident, test):
        # C.f.: find_result()
        key = test.get('find-result')
        if not self._validation:
            info("DUMP: %s" % self.test_name(ident, test, ["Find result", key]))
        result, rc, output = 0, 0, ['dump']
        self.save_result(ident, test, rc, output)
        return result

    def dump_include(self, ident, test):
        # C.f.: include()
        result = 0
        filename = test.get('include')
        filepath = os.path.join(self._include_dir, filename)
        if not self._validation:
            info("DUMP: %s" % self.test_name(ident, test, ["Include", filename]))
        try:
            data = self.load(filepath)
            result = self.validate(filepath, data, includefile=True)
            if result:
                return result
            if not self._validation:
                self.print_meta(filepath, data)
            result = self.dump_file(filepath, data)
        except TestsImportException as e:
            if self._stop_on_fail:
                raise
            exception("Exception occurred parsing %s:" % filepath)
            result = 1
        return result

    def dump_gconfig(self, ident, test):
        gctree = test.get('gconfig-tree', "hwinventory")
        gcvar = test.get('gconfig')
        command = "gconfig: %s: %s" % (gctree, gcvar)
        if not self._validation:
            info("DUMP: %s" % self.test_name(ident, test, command))
        result, rc, output = 0, 0, ['dump']
        self.save_result(ident, test, rc, output)
        return result

    def dump_gconfig_list(self, ident, test):
        gclist = test.get('gconfig-list', [])
        command = "gconfig: list of %d entries..." % (len(gclist))
        if not self._validation:
            info("DUMP: %s" % self.test_name(ident, test, command))
        result = 0
        return result

    def dump_python(self, ident, test):
        command = self.test_python(ident, test)
        if not self._validation:
            info("DUMP: %s" % self.test_name(
                ident, test, ['python', command.replace('\n', '; ')]))
        result, rc, output = 0, 0, ['dump']
        self.save_result(ident, test, rc, output)
        return result

    def dump_placeholder(self, ident, test):
        placeholder = test.get('placeholder')
        if not isinstance(placeholder, bool) or placeholder != True:
            self.warning(
                "%s: Invalid placeholder: Expected boolean 'True'" %
                (self.test_name(ident, test)))
        if not self._validation:
            info("DUMP: %s" % self.test_name(ident, test, 'placeholder'))
        return 0

    def conditional_dump(self, ident, test, conditionals):
        # C.f.: conditional()
        if not conditionals:
            return True
        if isinstance(conditionals, dict):
            conditionals = [conditionals]
        map(lambda cond: self.cond_dump(ident, test, cond), conditionals)

    def cond_dump(self, ident, test, cond):
        # C.f.: cond_check()
        if not self._validation:
            info("%s: Conditional Check: %s, %s" %
                 (self.test_name(ident, test), cond, cond.get('key')))

# #############################################################################

def default_check():
    tests = Tests(dump_saved_results=pargs.savedkeys)
    if pargs.dump:
        result = tests.dump()
    else:
        result = tests.execute()
    return result

def alternate_check(input_file):
    tests = Tests(input_file, dump_saved_results=pargs.savedkeys)
    if pargs.dump:
        result = tests.dump()
    else:
        result = tests.execute()
    return result

def hwcheck():
    fail("\"hwcheck\" option is not yet supported")
    return 1

# #############################################################################

parser = argparse.ArgumentParser(
    description=('Platform Check Framework. Executes a set of platform specific '
                 'checks of configuration and hardware. Optionally runs '
                 'user-provided platform check specification file.'))

parser.add_argument('-d', '--debug', action='count',
                    help='Enable (increase) debug output')
parser.add_argument('-v', '--verbose', action='count',
                    help='Enable (increase) verbose output')

parser.add_argument('--stoponfail', action='store_true',
                    help='Stop immediately on first failure')

group = parser.add_mutually_exclusive_group()
group.add_argument('--hwcheck', action='store_true',
                    help='Alternate short hardware check test specification')
group.add_argument('--input',
                    help='Alternate input filename with test specification',
                    default=DEFAULT_input_filename)

group = parser.add_mutually_exclusive_group()
group.add_argument('--syslog', action='store_true',
                    help='Route all output logging to syslog')
group.add_argument('--logfile',
                    help='Change output logfile from default: %s' %
                    DEFAULT_log_filename,
                    default=DEFAULT_log_filename)

parser.add_argument('--logcount',
                    help='Change output logfile count from default: %s' %
                    DEFAULT_log_backupCount,
                    default=DEFAULT_log_backupCount)

parser.add_argument('--dump', action='store_true',
                    help=('Dump test specification file (after parse and validate) '
                          '(do not execute)'))
parser.add_argument('--savedkeys', action='store_true',
                    help='Dump saved keys at end of test')

# #############################################################################

def main():
    global log
    global pargs

    start = end = 0
    result = 0
    pargs = parser.parse_args()
    log = init_logging()

    if pargs.dump:
        # Bump up VERBOSE level
        if not pargs.verbose:
            pargs.verbose = 1
    if pargs.debug:
        set_log_levels({'disp':logging.DEBUG, 'file':logging.DEBUG})

    try:
        start = time.time()
        info("START OF %s: %s" % (app_name.upper(), time.ctime(start)))
        if pargs.hwcheck:
            result = hwcheck()
        elif pargs.input:
            result = alternate_check(pargs.input)
        else:
            result = default_check()

    except TestsImportException as e:
        exception("Test Configuration File Import Exception occurred:")
        result = 1
    except TestsExecException as e:
        exception("Test Execution Exception occurred:")
        result = 1
    except TestsException as e:
        exception("Unhandled Test Exception occurred:")
        result = 1
    except Exception as e:
        #if pargs.debug:
        #    raise
        exception("Unhandled Exception occurred:")
        result = 1

    finally:
        end = time.time()
        info("END OF %s: %s" % (app_name.upper(), time.ctime(end)))

    e = str(datetime.timedelta(seconds=int(end - start)))
    if result:
        fail("Failed %s: result = %s, elapsed = %s" % (app_name, result, e))
    else:
        info("Passed %s: result = %s, elapsed = %s" % (app_name, result, e))
    return result

if __name__ == '__main__':
    sys.exit(main())
