#!/usr/bin/env python

# This script is a wrapper script for isi patch install of DSP packages.
# Refer to the usage message for more details.
# NB: Pre-8.0 Riptide, OneFS releases used 'isi pkg' rather than the
# 'isi patch' command. These two isi_dsp_install scripts are incompatible,
# although they both consume identical 'isi pkg' style DSP packages.

# Copyright C 2008-2015 by Isilon Systems LLC. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
# 
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
# 
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems.
# 
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.

##############################################################################

import getopt
import glob
import os
import os.path
import re
import subprocess
import sys
import tempfile
import time

# Using Isilon isi.misc for patch APIs. This breaks earlier Specs/Requirements
# and design goals of making a isi_dsp_install wrapper script that is 100%
# independnet of OneFS Python (thus inline subprocess APIs), but this way we
# are independent of any 'isi patch' CLI changes in the future.
import isi.misc

# Using Isilon isi.papi to get NDU upgrade status
import isi.papi.basepapi as papi
from isi.papi.papicommon import map_status, STATUS

##############################################################################

usage_msg = """\
Usage: %s [OPTIONS] [PACKAGE]

Updates the cluster drive_config.gc data with DSP package information.
With no options specified, the default behavior is:
  1. Verify no other DSP packages are installed
  2. Install the package
  3. Verify the package is installed
  4. Uninstall the package
  5. Verify the package installation (check logs for errors)

OPTIONS:
  --delete         Attempt package deletion     (Do not install)
  --check          Attempt install verification (Do not install)

  --save           Do not delete the temporary concatenated logfile
  --start [VALUE]  Set the start time for install logfile verification
                   Acceptable time value formats:
                   - Integer Unix time value
                   - "YYYY-MM-DD HH:MM:SS"

  --latest         Display latest DSP install time and package name
  --history        Display history of DSP installations

  -d, --debug      Increase debug level
  -v, --verbose    Increase verbosity level
  -q, --quiet      Decrease verbosity level to 0
  -h, --help       Display this help message
""" % os.path.basename(sys.argv[0])

##############################################################################

DEFAULT_LOGFILE = "isi_dsptool"
DEFAULT_OUTFILE = None # Use mkstemp for concatenate_logfiles
DEFAULT_TEMPDIR = "/var/crash"
HISTORY_LOGFILE = "/ifs/.ifsvar/dsp_install_history"
HISTORY_RECORD_LIMIT = 30 # Max number of install history records to keep
NDU_UPGRADE_STATUS = "/3/upgrade/cluster"

options = {
    'logfile' : DEFAULT_LOGFILE,
    'outfile' : DEFAULT_OUTFILE,
    'tempdir' : DEFAULT_TEMPDIR,
    'savelog' : False,
    'package' : None,
    'start'   : None, # Verification start time, set by code
    'verbose' : 1,    # Integer level
    'debug'   : 0,    # Integer level
    'install' : True, # Actions
    'verify'  : True, # Actions
    'delete'  : True, # Actions
    'latest'  : False,
    'history' : False,
    }

##############################################################################

def error(msg):
    if isinstance(msg, (list, tuple)):
        for m in msg:
            print >> sys.stderr, " - %s" % m
    else:
        print >> sys.stderr, "ERROR: %s" % msg

def warning(msg):
    if isinstance(msg, (list, tuple)):
        for m in msg:
            print >> sys.stderr, " - %s" % m
    else:
        print >> sys.stderr, "WARNING: %s" % msg

def verbose(level, msg):
    lvl = options.get("verbose", 0)
    dbg = options.get("debug", 0)
    if lvl > level or dbg:
        if isinstance(msg, (list, tuple)):
            for m in msg:
                print(" - %s" % m)
        else:
            print(msg)

def debug(level, msg):
    lvl = options.get("debug", 0)
    if lvl > level:
        if isinstance(msg, (list, tuple)):
            for m in msg:
                print(" - %s" % m)
        else:
            print(msg)

def quote(qstr):
    if '"' in qstr:
        q = "'%s'" % qstr.strip("'")
    else:
        q = '"%s"' % qstr.strip('"')
    return q

def system_command_output(args, shell=True, ignore_stderr=False):
    pipe = subprocess.PIPE
    # Due to annoyances with '*' wildcards, and shell=True or False, etc,
    # always pass a string to Popen.
    if isinstance(args, basestring):
        cmd = args
    else:
        cmd = ' '.join(args)
    p = subprocess.Popen(cmd, stdout=pipe, stderr=pipe,
                         close_fds=True, shell=shell)
    o, e = p.communicate()
    rc = p.returncode or (not ignore_stderr and e) and 1 or 0
    return (rc, filter(None, e.split('\n')), filter(None, o.split('\n')))

def system_command(cmd, estring, vlevel=None):
    rc, err, out = system_command_output(cmd)
    if rc or err:
        error("%s: rc=%d%s" % (estring, rc, rc and ':' or ''))
        error(err + out)
        return 1
    if vlevel is not None:
        verbose(vlevel, out)
    return 0

def copy_file(source, target):
    return system_command(
        ['cp', '-fp', quote(source), quote(target)],
        "Unable to copy %s to %s:" % (source, target))

def unzip_file(zipfile):
    return system_command(
        ['gunzip', quote(zipfile)],
        "Unable to unzip %s:" % (zipfile))

def concat_files(source, target):
    return system_command(
        ['cat', quote(source), '>>', quote(target)],
        "Unable to cat %s to %s:" % (source, target))

def remove_file(file):
    return system_command(
        ['rm', '-rf', quote(file)],
        "Unable to remove file %s:" % (file))

def remove_dir(dir):
    return system_command(
        ['rm', '-rf', quote(dir)],
        "Unable to remove dir %s:" % (dir))

def isi_pkg_command(action, package, retries=None, delay=None):
    if not retries:
        if action in ['install', 'delete']:
            # None on success, string error message on failures.
            os.environ["PKG_ALLOW_RESTRICTED"] = "true"
            rc = isi.misc.pkg_request(action, package, False, False, False)
            if rc:
                error("Patch %s %s: rc=%s" % (action, package, rc))
                return 1
            return 0
        raise Exception("code error: invalid action %s" % action)
    for loop in xrange(retries):
        loop and time.sleep(delay or 1)
        rc = isi_pkg_command(action, package)
        if rc == 0:
            break
        warning("Retry Loop %d: isi patch %s failed" % (loop+1, action))
    return rc

def isi_pkg_info(pkgname):
    # None on error
    return isi.misc.pkg_details(pkgname, False)

def isi_pkg_list():
    # NOTE: If the previous package install has not completed,
    # then the package will NOT show up in pkg_list_installed -
    # but it WILL show up with pkg_details(packagename).
    return dict([(pkg, isi.misc.pkg_details(pkg, False))
                 for pkg in isi.misc.pkg_list_installed()])

# Poll on isi patch status output until desired status.
# 10 minutes should be more than enough time, but not too short.
# We can use a short delay=1 sec to be more responsive, the only
# cost is if running in verbose mode (level 2), extra command output.
# Typical times for Riptide, DSP v1.5, approx 100 models and 13 vendors,
# is 40-45 loops @ 1 sec each for both install and delete.
# I.e., "approximately one minute," expected.

def isi_pkg_info_poll_loop(pkgname, desired_status, max_wait=10*60, delay=1):
    rc = 1 # Assume failed, not desired Install Status, until otherwise
    status = None # Unknown
    def poll_warn(loop, what):
        warning("Polling Loop %02d: Unable to %s" % (loop+1, what))
    for loop in xrange(max_wait / delay):
        time.sleep(delay)
        verbose(2, "Polling Loop %02d:" % (loop+1))
        try:
            info = isi_pkg_info(pkgname)
        except:
            poll_warn(loop, "get DSP package installation status")
            continue
        if not info:
            poll_warn(loop, "get DSP package installation status")
            continue
        patchstatus = info.get('status')
        if not patchstatus:
            poll_warn(loop, "determine DSP package installation status")
        if patchstatus and patchstatus != status:
            status = patchstatus
            verbose(1, "Polling Loop %02d: "
                    "DSP Install Status changed to %s" % (loop+1, status))
        if status == desired_status:
            rc = 0
            break
    return rc

##############################################################################

severity_list = [
    'EMERGENCY', 'ALERT', 'CRITICAL', 'ERROR',
    'WARNING', 'NOTICE', 'INFO', 'DEBUG',
    ]

severity_short = {
    'CRIT': 'CRITICAL',
    'WARN': 'WARNING',
    }

severity_values  = dict(zip(*(severity_list, range(len(severity_list)))))

def severity_value(severity):
    sev = None
    if severity is None or isinstance(severity, int):
        return severity
    if not isinstance(severity, basestring):
        error("Invalid severity %s, type %s" % (severity, type(severity)))
    else:
        try:
            sev = int(severity)
        except:
            if severity in severity_short:
                sev = severity_values.get(severity_short[severity])
            else:
                sev = severity_values.get(severity)
    return sev

severity_strings = dict(zip(*(range(len(severity_list)), severity_list)))

def severity_string(severity):
    return severity_strings.get(severity) or 'UNKNOWN'

# Simple Test:
# tin  = [None, 0, 1, 3, '0', '1', '3', 'EMERGENCY', 'ALERT', 'ERROR',
#        'CRIT', 'WARN', 'FOO', 3.14]
# tout = [None, 0, 1, 3, 0, 1, 3, 0, 1, 3, 2, 4, None, None]
# map(severity_value, tin) == tout

##############################################################################

# Syslog RFC 5424: https://datatracker.ietf.org/doc/rfc5424/

# "alt1" is for newsyslog rollovers: e.g.:
# Nov 13 19:30:00 erb1-1 newsyslog[83959]: logfile turned over due to size>500K

rfc5424_fulldate   = (r"(?P<year>\d{4})-(?P<mon>\d{2})-(?P<day>\d{2})")
rfc5424_alt1date   = (r"([A-Z][a-z]{2})\s+(?P<day>\d{1,2})")
rfc5424_parttime   = (r"(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})(?P<fracsec>\.\d+)?")
rfc5424_timeoffset = (r"[Z]?(?P<offset>-\d{2}:\d{2})?")
rfc5424_fulltime   = (r"%s%s" % (rfc5424_parttime, rfc5424_timeoffset))
rfc5424_timestamp  = (r"%s[T ]%s" % (rfc5424_fulldate, rfc5424_fulltime))
rfc5424_time_alt1  = (r"%s\s+%s" % (rfc5424_alt1date, rfc5424_fulltime ))
rfc5424_pri        = (r"<(?P<facility>\d+)\.(?P<severity>\d+)>")
rfc5424_prilevel   = (r"(?P<level>[A-Z]+)")
rfc5424_priority   = (r"((%s)|(%s))" % (rfc5424_pri, rfc5424_prilevel))
rfc5424_hostname   = (r"(?P<hostname>\S+)")
rfc5424_appname    = (r"(?P<appname>[^[]+)")
rfc5424_procid     = (r"(?P<procid>\d+)")
rfc5424_id         = (r"(%s\s+%s\[%s\]:)?" %
                      (rfc5424_hostname, rfc5424_appname, rfc5424_procid))

isi_debugfuncline  = (r"(?P<func>[^:]+):(?P<line>\d+):")
isi_pkgscript      = (r":\s*(?P<script>[^:]+):")
isi_msg            = (r"((%s)|(%s))?\s*(?P<msg>.*)" %
                      (isi_debugfuncline, isi_pkgscript))

rfc5424_re = re.compile(
    r"%s\s+%s\s*%s\s*%s$" %
    (rfc5424_timestamp, rfc5424_priority, rfc5424_id, isi_msg))

rfc5424_re_numeric_keys = [
    'year', 'mon', 'day', 'hour', 'min', 'sec', 'fracsec',
    'facility', 'severity', 'procid', 'line',
    ]

rfc5424_alt1_re = re.compile(
    r"%s\s+%s?\s*%s\s*%s$" %
    (rfc5424_time_alt1, rfc5424_priority, rfc5424_id, isi_msg))

rfc5424_alt1_re_missing_keys = {
    'year' : time.localtime().tm_year,
    'mon'  : time.localtime().tm_mon,
    }

def parse_syslog_line(line):
    r = {}
    match = rfc5424_re.match(line.strip())
    if match:
        r = match.groupdict()
    else:
        match = rfc5424_alt1_re.match(line.strip())
        if match:
            r = match.groupdict()
            r.update(rfc5424_alt1_re_missing_keys)
            if not r['level']:
                r['level'] = "INFO"
    return r

def parse_syslog_lines(lines):
    data = [(c, line.strip(), parse_syslog_line(line))
            for (c, line) in zip(*(range(len(lines)), lines))]
    invd = filter(lambda d: not d[2], data)
    try:
        data = zip(*filter(lambda d: d[2], data))[2]
    except IndexError:
        data = []
    if invd:
        warning("Unable to parse %d lines in logfile" % len(invd))
        w = len("%s" % invd[-1][0])
        warning(["[L%0*d] %s" % (w, c, line) for (c, line, d) in invd])
    for d in data:
        if d['severity'] is None and d['level']:
            d['severity'] = severity_value(d['level'])
        for key in rfc5424_re_numeric_keys:
            if d[key] is not None:
                d[key] = int(d[key])
        try:
            d['time'] = mktime_logtime(d)
        except ValueError:
            # If we replaced the month in parse_syslog_line, we can end up with
            # an invalid month / day combination with more days than we have in
            # the month. In this case, make day = 28 to produce valid dates
            if d['day'] > 28 and d['day'] < 32:
                d['day'] = 28
            d['time'] = mktime_logtime(d)
    return data

ERROR = severity_value('ERROR')
WARNING = severity_value('WARNING')

error_to_warning_msg_regexes = [
    re.compile('.*parse warnings.*dropping unrecognized entries.*'),
    ]

def filter_errors(data):
    # Catch any matching "error-to-warning" cases (report as warnings)
    result = filter(lambda d: d['severity'] <= ERROR, data)
    for regex in error_to_warning_msg_regexes:
        result = filter(lambda d: not regex.match(d['msg']), result)
    return result

def filter_warnings(data):
    # Catch any matching "error-to-warning" cases (report as warnings)
    # We could trivially do a separate filter, appending these at the end,
    # or we could walk the data, maintaining original order. Do the latter.
    result = []
    for d in data:
        if d['severity'] == WARNING:
            result.append(d)
        elif d['severity'] <= ERROR:
            for regex in error_to_warning_msg_regexes:
                if regex.match(d['msg']):
                    d['severity'] = WARNING
                    d['level'] = 'WARNING'
                    result.append(d)
                    break
    return result

def filter_non_informational(data):
    return filter(lambda d: d['severity'] <= WARNING, data)

def filter_informational(data):
    return filter(lambda d: d['severity'] > WARNING, data)

##############################################################################

fmt_time_str = "%Y-%m-%d %H:%M:%S"

def format_time(tsec):
    return time.strftime(fmt_time_str, time.localtime(tsec))

def format_logtime(data):
    Y, m, d = data['year'], data['mon'], data['day']
    H, M, S = data['hour'], data['min'], data['sec']
    return "%04d-%02d-%02d %02d:%02d:%02d" % (Y, m, d, H, M, S)

def mktime_logtime(data):
    return int(time.mktime(time.strptime(format_logtime(data), fmt_time_str)))

def mktime_string(tstring):
    mktime_fmt_time_strings = [
        "%Y-%m-%d %H:%M:%S",
        "%Y-%m-%dT%H:%M:%S",
        "%Y-%m-%d %H:%M",
        "%Y-%m-%dT%H:%M",
        "%Y-%m-%d",
        ]
    t = None
    try:
        t = int(tstring)
    except ValueError:
        for fmt in mktime_fmt_time_strings:
            try:
                t = int(time.mktime(time.strptime(tstring, fmt)))
                break
            except:
                t = None
        if t is None:
            error("Invalid time string: %s does not match format %s" %
                  (tstring, fmt_time_str))
    return t

##############################################################################

def format_logdata(data, fmt_options={}):
    ftime = flvl = fpri = fhost = fapp = ffunc = fscript = ""
    if fmt_options.get("timestamp"):
        ftime = format_logtime(data) + " "
    if fmt_options.get("level"):
        lvl = data['level'] or severity_string(data['severity'])
        # Use width 5 'minimum', since INFO and DEBUG are common.
        # Also, consistent with any ERRORs.
        flvl  = ("%-5s " % lvl)
    if fmt_options.get("priority"):
        fac, sev = data['facility'] or '?', data['severity']
        fpri  = ("<%s.%s> " % (fac, sev))
    if fmt_options.get("hostname"):
        host = data['hostname']
        fhost = (host and "%s " % (host) or "")
    if fmt_options.get("appname"):
        app, proc = data['appname'], data['procid']
        fapp  = (proc and "%s[%s] " % (app, proc) or "")
    if fmt_options.get("script"):
        script = data['script']
        fscript = (script and "%s: " % (script) or "")
    if fmt_options.get("funcline"):
        func, line = data['func'], data['line']
        ffunc = (func and "%s:%s " % (func, line) or "")
    fmsg = data['msg']
    return ("%s%s%s%s%s%s%s%s" %
            (ftime, flvl, fpri, fhost, fapp, fscript, ffunc, fmsg))

def report_logdata_results(logdata):
    rc = 0
    logfile = options.get("logfile")
    vlevel = options.get("verbose", 0)
    dlevel = options.get("debug", 0)
    start = options.get("start")
    fmt_options = {
        'timestamp'  : dlevel > 0 or vlevel > 1,
        'script'     : dlevel > 0 or vlevel > 2,
        'funcline'   : dlevel > 1,
        'priority'   : dlevel > 2,
        'hostname'   : dlevel > 2,
        'appname'    : dlevel > 2,
        'level'      : True,
        }
    fmt_options.update(options)
    if start:
        fstart = format_time(start)
        fdata = filter(lambda d: d['time'] >= start, logdata)
        if fdata:
            verbose(2, "Filter logdata to %s or later: %d of %d lines remain" %
                    (fstart, len(fdata), len(logdata)))
            logdata = fdata
        else:
            error("No logdata lines for %s or later" % fstart)
            return 1
    errors = filter_errors(logdata)
    warnings = filter_warnings(logdata)
    extras = dlevel and filter_informational(logdata) or []
    if (errors):
        rc = 1
        error("Found %s error messages in %s logfile" %
               (len(errors), logfile))
    if (warnings):
        warning("Found %s warning messages in %s logfile" %
                (len(warnings), logfile))
    if (extras):
        verbose(0, "Found %s informational messages in %s logfile" %
                (len(extras), logfile))
    if dlevel:
        outdata = logdata
    elif vlevel:
        outdata = filter_non_informational(logdata)
    else:
        outdata = []
    verbose(0, [format_logdata(data, fmt_options) for data in outdata])
    return rc

##############################################################################

# Observed:
# If there are already unzipped log files, then syslog will in fact overwrite
# them with new .gz zip files, BUT ONLY FOR N=0. It leaves N=1+ alone. Odd.
# I.e., if we start with this:    We end up with this, later:
# /var/log/isi_dsptool.log        /var/log/isi_dsptool.log
# /var/log/isi_dsptool.log.0      /var/log/isi_dsptool.log.0.gz
# /var/log/isi_dsptool.log.1      /var/log/isi_dsptool.log.1.gz
# /var/log/isi_dsptool.log.2      /var/log/isi_dsptool.log.2.gz
#                                 /var/log/isi_dsptool.log.1
#                                 /var/log/isi_dsptool.log.2
# where the latter two ".1, .2" files are the original unzipped files.
# We have lost the .0, but kept the .1 and .2.
# Conclusion: IGNORE any and all unzipped ".log.N" files, for safety.
# ONLY look at the "main" .log file, and any .log.N.gz zipped files.

def concatenate_logfiles():
    logfile = options.get("logfile")
    outfile = options.get("outfile")
    tempdir = options.get("tempdir")
    logdir = "/var/log"
    prefix = "%s." % logfile
    mainfile = os.path.join(logdir, "%s.log" % logfile)
    zipfiles = map(os.path.basename, glob.glob("%s.*.gz" % mainfile))
    if not zipfiles and not os.path.exists(mainfile):
        error("No %s logfiles exist for parsing" % logfile)
        return None
    if outfile:
        if os.path.exists(outfile):
            verbose(0, "Overwriting %s..." % outfile)
            remove_file(outfile)
    else:
        (fd, outfile) = tempfile.mkstemp(prefix=prefix, dir=tempdir, text=True)
    cptmpdir = tempfile.mkdtemp(prefix=prefix, dir=tempdir)
    try:
        for zipfile in zipfiles:
            copy_file(os.path.join(logdir, zipfile), cptmpdir)
            unzip_file(os.path.join(cptmpdir, zipfile))
            source = os.path.join(cptmpdir, zipfile).replace('.gz', '')
            concat_files(source, outfile)
    finally:
        remove_dir(cptmpdir)
    if not os.path.exists(mainfile):
        warning("Main %s logfile not found" % mainfile)
    else:
        concat_files(mainfile, outfile)
    return outfile

##############################################################################

def get_syslog_data():
    logfile = options.get("logfile")
    outfile = concatenate_logfiles()
    if not outfile:
        error("Unable to get syslog data for %s" % logfile)
        return None
    try:
        with open(outfile) as fd:
            data = parse_syslog_lines(fd.readlines())
        if not data:
            error("Unable to get syslog data for %s" % logfile)
            data = None
    finally:
        if options.get("savelog"):
            verbose(0, "%s logfile data available at %s" % (logfile, outfile))
        else:
            remove_file(outfile)
    return data

##############################################################################

def dsp_verify_install():
    data = get_syslog_data()
    if not data:
        return 1
    return report_logdata_results(data)

##############################################################################

DSP_ISI_PKG_EXTS = [ ".tar", ".tar.gz", ".tgz" ]

def dsp_pkgname(package):
    pkgname, pkgext = os.path.basename(package), None
    for ext in DSP_ISI_PKG_EXTS:
        if pkgname.endswith(ext):
            pkgname, pkgext = pkgname.replace(ext, ''), ext
            break
    return pkgname, pkgext

def dsp_valid_package(package, exists=True, tarfile=True):
    if not package:
        error("No package specified")
        return False
    if exists:
        if not os.path.exists(package):
            error("Package does not exist: %s" % package)
            return False
    if tarfile:
        pkgname, pkgext = dsp_pkgname(package)
        if not pkgext in DSP_ISI_PKG_EXTS:
            error("Invalid package, expected isi pkg tar file: %s" % package)
            return False
    return True

# With Riptide 'isi patch', asynchronous installation process, cannot
# assume installation is COMPLETE after running 'install' command.
# Poll for completion.

def dsp_install():
    rc = 0
    package = options.get("package")
    if not dsp_valid_package(package):
        return 1
    pkgname, pkgext = dsp_pkgname(package)
    verbose(1, "Installing %s ..." % pkgname)
    rc = isi_pkg_command('install', package)
    if rc:
        return rc
    verbose(1, "Waiting for %s installation to complete ..." % pkgname)
    rc = isi_pkg_info_poll_loop(pkgname, "Installed")
    if rc == 0:
        verbose(0, "Installed %s" % pkgname)
    return rc

# With Riptide 'isi patch', asynchronous deletion process, cannot
# assume deletion is COMPLETE after running 'delete' command.
# Poll for completion.

# With Riptide 'isi patch' system, issuing a 'delete' command immediately
# after we have gone to "Status: Installed" will fail sometimes (all
# the time?), with "Upgrade may already be in progress, create failed,
# failed to write config file" reported errors.
# Simply use a retry loop here, with a fixed delay up front.
# Trying numbers less than 30 (e.g., 5) didn't help much -- it would
# still fail the first, and successive retry loop attempts (with delays
# of 5 seconds each) - even if the total loop time exceeded 30 seconds.
# And yet if I delay 30 seconds up front, it seemed to work better. YMMV.
# In case this is exacerbated on large clusters with other 'isi patch'
# activity, be sure to use a long enough retry loop: 12x5sec=1 minute?

def dsp_uninstall(package):
    rc = 0
    pkgname, pkgext = dsp_pkgname(package)
    delay = 30
    if options.get("install") and delay:
        verbose(1, "Delaying %d seconds after Install before Delete attempt" %
                delay)
        time.sleep(delay)
    verbose(1, "Removing %s package files..." % pkgname)
    rc = isi_pkg_command('delete', pkgname, retries=720, delay=5)
    if rc:
        return rc
    verbose(1, "Waiting for %s deletion to complete ..." % pkgname)
    rc = isi_pkg_info_poll_loop(pkgname, "None")
    if rc == 0:
        verbose(0, "Removed %s package files" % pkgname)
    return rc

# We can check for the explicit 'package' specified -- but when we do a
# normal install sequence (see main), this is the filename, which MAY be
# different from the internal DSP Package Name (as specified by the XML).
# The user may have copied and renamed a package. This is more expected
# for internal use, rather than normal external customer use case.
# Either way, regardless, the 'package' (filename) may be different.
# Let's make some ASSUMPTIONS, and do our best effort:
# 1. DSP package files (and names) ALWAYS start with "Drive_Support_".
# 2. ONLY ONE DSP package may be installed at any time.

def dsp_installed():
    installed = None
    install_status = "Unknown Install Status"
    package = options.get("package")
    if package:
        if not dsp_valid_package(package, exists=False):
            return False
        pkgname, pkgext = dsp_pkgname(package)
        verbose(1, "Checking if %s is installed ..." % pkgname)
    else:
        verbose(1, "Checking if DSP package is installed ...")
    info = isi_pkg_list()
    if package and pkgname in info:
        installed = pkgname
        install_status = info[installed].get('status')
    if not installed:
        dsps = filter(lambda s:s.startswith("Drive_Support_"), info)
        if dsps: # Just grab the first one. Don't expect 2+ anyway.
            installed = dsps[0]
            install_status = info[installed].get('status')
    if installed:
        verbose(1, "Package %s is %s" % (installed, install_status))
    elif package:
        verbose(1, "Package %s is NOT installed" % pkgname)
    else:
        verbose(1, "No DSP packages are installed")
    return installed

##############################################################################

def usage(rc):
    print usage_msg
    sys.exit(rc)

def init_options(args):
    short = "s:dqvh"
    long = [
        "delete", "check", "save", "start=", "debug",
        "quiet", "verbose", "latest", "history", "help",
        ]
    help = quiet = False
    try:
        opts, args = getopt.getopt(args, short, long)
        if args:
            options["package"] = args.pop(0)
        if args:
            raise getopt.GetoptError("Unknown arguments: %s" % ' '.join(args))
        for opt, arg in opts:
            o = opt.strip('-')
            if o in ["delete"]:
                options.update(
                    {"install": False, "verify": True, "delete": True })
            elif o in ["check"]:
                options.update(
                    {"install": False, "verify": True, "delete": False})
            elif o in ["save"]:
                options["savelog"] = True
            elif o in ["s", "start"]:
                try:
                    options["start"] = mktime_string(arg)
                except:
                    raise getopt.GetoptError("Invalid start value: %s" % arg)
            elif o in ["d", "debug"]:
                options["debug"] += 1
            elif o in ["q", "quiet"]:
                quiet = True
            elif o in ["v", "verbose"]:
                options["verbose"] += 1
            elif o in ["latest"]:
                options.update({"install": False, "verify": False,
                                "delete": False, "latest": True})
            elif o in ["history"]:
                options.update({"install": False, "verify": False,
                                "delete": False, "history": True})
            elif o in ["h", "help"]:
                help = True
        if options["start"] and options["install"]:
            raise getopt.GetoptError(
                "'--start' option is incompatible with package install")
        if options["package"] and not options["install"]:
            raise getopt.GetoptError(
                "'--check' option is incompatible with specified package")
    except getopt.GetoptError, e:
        error("Invalid options: %s" % str(e.msg))
        usage(1)
    if quiet:
        options["verbose"] = 0
    if help:
        usage(0)

def get_action():
    if options.get("install"):
        action = "Install"
    elif not options.get("delete"):
        action = "Verify"
    else:
        action = "Delete"
    return action

def get_etime():
    # Some cases (--check) don't always set start time
    if options.get("start"):
        etime = time.time() - options.get("start")
        return "%s sec elapsed" % (etime)
    return None

def result_string(rc):
    status = rc and "Failed" or "Succeeded"
    action = get_action()
    if action in ["Verify"]:
        return "DSP %s %s" % (action, status)
    else:
        return "DSP %s %s, %s" % (action, status, get_etime())

def read_install_history():
    if os.path.isfile(HISTORY_LOGFILE):
        with open(HISTORY_LOGFILE, "r") as fin:
            lines = fin.readlines()
            lines = [l.strip() for l in lines if l != '\n']
            if len(lines) > 0:
                return lines
    return ["None"]

def write_install_history(dsp_name, dsp_time):
    lines = []
    if os.path.isfile(HISTORY_LOGFILE):
        with open(HISTORY_LOGFILE, "r") as fin:
            lines = fin.readlines()
            lines = [l.strip() for l in lines if l != '\n']
    lines.append("%s || %s" % (dsp_time, dsp_name))
    if len(lines) > HISTORY_RECORD_LIMIT:
        lines = lines[-HISTORY_RECORD_LIMIT:]
    lines.append('\n')
    with open(HISTORY_LOGFILE, "w") as fout:
        fout.write('\n'.join(lines))

# Get NDU availability status
# Returns True if available, False if busy, and None on errors
def is_ndu_available():
    try:
        response = papi.get(NDU_UPGRADE_STATUS)
        if map_status(response.status) != STATUS.SUCCESS:
            return None
    except:
        return None

    if response.body['cluster_state'] == 'committed' and response.body['current_process'] == None:
        return True
    return False

def main():
    rc = 0
    skip_verify = False
    skip_result_print = False
    verbose(2, "DSP %s Start: %s" % (get_action(), time.ctime()))

    ndu_status = is_ndu_available()
    if ndu_status is None:
        error("Unable to determine cluster status. Please try again.");
        return 1
    elif ndu_status == False:
        error("The upgrade system is busy processing another request.");
        return 1

    if options.get("install"):
        if not options["start"]:
            options["start"] = time.time()
        if not dsp_valid_package(options.get("package")):
            return 1
        installed = dsp_installed()
        if installed:
            error("%s is already installed" % installed)
            return 1
        if dsp_install():
            rc = 1
        else: # Save a record of the install
            dsp_name = os.path.basename(options["package"])
            dsp_time = time.strftime(fmt_time_str, time.localtime())
            write_install_history(dsp_name, dsp_time)
    if options.get("delete"):
        if not options["start"]:
            options["start"] = time.time()
        installed = dsp_installed()
        if not installed:
            if options.get("install"):
                warning("%s is not installed" % options.get("package"))
                rc = 1
            else:
                # No install - must have been a delete.
                # If no packages installed, no need to verify.
                skip_verify = True
        if installed:
            if dsp_uninstall(installed):
                rc = 1
    if options.get("verify") and not skip_verify:
        if dsp_verify_install():
            rc = 1
    if options.get("latest"): # Print record of latest install (if it exists)
        verbose(0, read_install_history()[-1])
        skip_result_print = True
    if options.get("history"): # Print dsp install history (if it exists)
        for line in read_install_history():
            verbose(0, line)
        skip_result_print = True
    if not skip_result_print:
        verbose(0, result_string(rc))
    return rc

if __name__ == '__main__':
    init_options(sys.argv[1:])
    rc = main()
    sys.exit(rc)

