#!/usr/bin/python

from __future__ import (print_function,
                        absolute_import,
                        division)
from datetime import datetime
import json
import logging
import optparse
import os
import shutil
import sys
import sqlite3
import tarfile
import traceback

ARCHIVE_DIR = "/ifs/.ifsvar/upgrade/archive"

# This directory will be temporarily hold the extracted files
# and/or databases from the archive tar until data collection
# from these items have completed. If multiple nodes run this
# tool on the same archive simultaneously, there will be a
# conflict which will result in a failure or corrupted output
TMP_DIR = "/ifs/.ifsvar/tmp"

RESULTS_FILE = "{archiveid}.json"
TIMESTAMP_FMT = "%Y-%m-%dT%H:%M:%S.%fZ"

AGENT_TIMESTAMP_FIELD = "WrittenTimeStamp"
ULOG_TIMESTAMP_FIELD = "TimeOccured"

# UpgradeLog-<node>.db queries
EVENTACTION_SCRIPT = '''
            SELECT TimeOccured, Entry \
            FROM LogEntry \
            WHERE Entry \
            LIKE "%/usr/share/upgrade/event-actions/{event_action}/%]" \
            ORDER BY TimeOccured \
            '''

# AgentPersistent.db queries
EVENTACTION = '''
            SELECT ROWID, WrittenTimeStamp, Command, Status \
            FROM CommandState \
            WHERE Command="{event_action}" \
            ORDER BY ROWID \
            '''

AGENT_PACKAGE = '''
            SELECT ROWID, WrittenTimeStamp, Command, Status \
            FROM CommandState \
            ORDER BY ROWID \
            '''

EVENT_ACTIONS_LIST = [
    {"agent": "HookPreUpgradeOptional",        "ulog": "pre-upgrade-optional"},
    {"agent": "HookPreUpgradeMandatory",       "ulog": "pre-upgrade-mandatory"},
    {"agent": "HookPendingReboot",             "ulog": "pending-reboot"},
    {"agent": "HookDataMigrationUpgrade",      "ulog": "data-migration-upgrade"},
    {"agent": "HookUpgradePostRebootInitial",  "ulog": "upgrade-post-reboot-initial"},
    {"agent": "HookPostReboot",                "ulog": "post-reboot"},
    {"agent": "HookUpgradePostRebootFinal",    "ulog": "upgrade-post-reboot-final"},
    {"agent": "HookLastNodePostUpgrade",       "ulog": "last-node-post-upgrade"},
    {"agent": "HookPreRollback",               "ulog": "pre-rollback"},
    {"agent": "HookDataMigrationRollback",     "ulog": "data-migration-rollback"},
    {"agent": "HookRollbackPostRebootInitial", "ulog": "rollback-post-reboot-initial"},
    {"agent": "HookRollbackPostRebootFinal",   "ulog": "rollback-post-reboot-final"},
    {"agent": "HookPostRollback",              "ulog": "post-rollback"},
    {"agent": "HookPostCommit",                "ulog": "post-commit"}
]

class Failure(Exception):
    pass

class DATABASE(object):
    def __init__(self):
        self._db = None
        self._db_name = None
        self._db_fname = None

    def connect(self, db_fname):
        self._db_fname = db_fname
        self._db_name = self._db_fname.split("/")[-1]
        self._db = sqlite3.connect(db_fname)
        self._db.row_factory = sqlite3.Row

    def close(self):
        self._db.close()

    def execute(self, query):
        rows = []
        try:
            cursor = self._db.execute(query)
            rows = cursor.fetchall()
            cursor.close()
        except Exception as ex:
            logger.exception("%s - %s" % (self._db_name, ex))
        finally:
            pass

        return rows

def add_agent_log(result_nodes_nid, hook, starttime="", endtime="", failed=False):
    """ Add entries of hook (event actions) from query to results """

    elapsed_time = calculate_elapsed(endtime, starttime)

    if result_nodes_nid["event_actions"].has_key(hook):
        result_nodes_nid["event_actions"][hook]["starttime"] = starttime
        result_nodes_nid["event_actions"][hook]["endtime"] = endtime
        result_nodes_nid["event_actions"][hook]["elapsed"] = elapsed_time
        result_nodes_nid["event_actions"][hook]["failed"] = failed
    else:
        result_nodes_nid["event_actions"][hook] = {
            "starttime": starttime,
            "endtime": endtime,
            "elapsed": elapsed_time,
            "failed": failed,
            "scripts": {}
        }

    return

def add_ulog_log(result_hook_scripts, rows):
    """ Add entries of hook scripts from query to results """

    def append_hook_script(scrname, start_entry, end_entry, raw, failure):
        elapsed = calculate_elapsed(end_entry["timestamp"], start_entry["timestamp"])
        result_hook_scripts[scrname] = {
            "starttime": start_entry["timestamp"],
            "endtime": end_entry["timestamp"],
            "raw": raw,
            "elapsed": elapsed,
            "failure": failure
            }

    # Organize rows by script names
    scripts = {}
    for row in rows:
        entry = row["Entry"]
        timestamp = row[ULOG_TIMESTAMP_FIELD]
        scrname = entry.split("/")[-1].rstrip("]")

        if scrname not in scripts:
            scripts[scrname] = []
        scripts[scrname].append({"entry": entry, "timestamp": timestamp})

    # Add group of logs according to script name to results
    for scrname, raw in scripts.items():
        start_entry = end_entry = {"entry": "", "timestamp": ""}
        for log in raw:
            if "Starting Hook [/usr/share/upgrade/event-actions/" in log["entry"]:
                start_entry = log
            elif "Hook Finished and Removed" in log["entry"]:
                end_entry = log
        failure = not end_entry["timestamp"] or "Failed" in str(raw) # failure if start, but no end

        append_hook_script(scrname, start_entry, end_entry, raw, failure)

    return

def calculate_elapsed(end, start):
    elapsed = str(datetime.strptime(end, TIMESTAMP_FMT) - \
        datetime.strptime(start, TIMESTAMP_FMT)) if (start and end) else ""

    return elapsed

def extract_member_tar(tar, mname):
    tar.extract(mname, TMP_DIR)

    extracted_file = os.path.join(TMP_DIR, mname)
    if not os.path.isfile(extracted_file):
        raise Failure("'{0}' does not exist -- possible error when extracting".format(extracted_file))

    extracted_member = os.path.join(TMP_DIR, mname)

    return extracted_member

def gather_from_db(result, db_fname, type_upgrade=True):
    if os.path.isfile(db_fname):
        try:
            db.connect(db_fname)

            if "UpgradeLog" in db_fname and type_upgrade:
                nid = os.path.basename(db_fname).rstrip(".db").split("-")[-1]
                gather_from_ulog(result, nid)
            elif "AgentPersistent" in db_fname:
                dbname_list = os.path.basename(db_fname).split("-")
                nid = dbname_list[-3]
                if type_upgrade:
                    gather_from_agent(result, nid)
                else:
                    gather_from_agent_pf(result, nid)
        except:
            logger.exception("Issue connecting or gathering from database: %s" % db_fname)
            raise
        finally:
            db.close()

    return

def gather_from_logfile(result, fname):
    """ Versions lower than 8.1.1.x do not use db for UpgradeLog. Use flat log file """

    if os.path.isfile(fname):
        nid = os.path.basename(fname).split("-")[1]
        with open(fname) as ulogf:
            rows = []
            for line in ulogf.readlines():
                tokens = line.split(" : ")
                timeoccured, entry = tokens[0].strip(), tokens[-1].strip()
                rows.append({ULOG_TIMESTAMP_FIELD: timeoccured, "Entry": entry})
            result["nodes"][nid] = result["nodes"].get(nid, {"event_actions": {}, "reboots": []})
            gather_from_ulog(result, nid, logf_logs=rows)

    return

def gather_from_agent(result, nid):
    """ Query the database for each event action and reboot and update the result """

    result["nodes"][nid] = result["nodes"].get(nid, {"event_actions": {}, "reboots": []})

    for event_action in EVENT_ACTIONS_LIST:
        hook_agent = event_action["agent"]

        try:
            hook_start_end = db.execute(EVENTACTION.format(event_action=hook_agent))

            if not hook_start_end: # No entry for hook in db
                add_agent_log(result["nodes"][nid], hook_agent)
            elif len(hook_start_end) == 1 and hook_start_end[0]["Status"] == "Start": # failure if start, but no end
                start_entry = hook_start_end[0]
                add_agent_log(result["nodes"][nid], hook_agent, starttime=start_entry[AGENT_TIMESTAMP_FIELD], \
                        endtime="", failed=True)
            else:
                start_entry = hook_start_end[0]
                end_fail_entry = hook_start_end[1]
                fail = True if end_fail_entry["Status"] == "Failed" else False
                add_agent_log(result["nodes"][nid], hook_agent, starttime=start_entry[AGENT_TIMESTAMP_FIELD], \
                        endtime=end_fail_entry[AGENT_TIMESTAMP_FIELD], failed=fail)
        except:
            logger.exception("Error gathering from AgentPersistent-%s" % nid)
            raise

    gather_reboot_results(result, nid)
    gather_agent_raw(result, nid)

    return

def gather_from_ulog(result, nid, logf_logs=None):
    """ Query the database for each hook script under an event action and update the result """

    result["nodes"][nid] = result["nodes"].get(nid, {"event_actions": {}, "reboots": []})

    from_logf = True if logf_logs else False

    for hook in EVENT_ACTIONS_LIST:
        hook_agent = hook["agent"]
        hook_ulog = hook["ulog"]

        if hook_agent not in result["nodes"][nid]["event_actions"]:
            add_agent_log(result["nodes"][nid], hook_agent)

        result_hook_scripts = result["nodes"][nid]["event_actions"][hook_agent]["scripts"]

        if not from_logf:
            scripts_list = db.execute(EVENTACTION_SCRIPT.format(event_action=hook_ulog))
        else:
            scripts_list = [line for line in logf_logs if hook_ulog in line["Entry"]]

        try:
            add_ulog_log(result_hook_scripts, scripts_list)
        except:
            logger.exception("Error gathering from UpgradeLog-%s " % nid)
            raise

    return

def gather_agent_raw(result, nid):
    """ Add raw log entries to result """

    try:
        query = "SELECT * FROM CommandState ORDER BY ROWID"
        rows = db.execute(query)

        result["nodes"][nid]["raw"] = ["{0}|{1}|{2}".format(
                                                    row["Command"],
                                                    row["Status"],
                                                    row[AGENT_TIMESTAMP_FIELD])
                                                    for row in rows]
    except:
        logger.exception("Error gathering raw data from AgentPersistent-%s" % nid)
        raise

def gather_from_agent_pf(result, nid):
    """ Query for package/firmware logs and update result """

    result["nodes"][nid] = result["nodes"].get(nid, {})

    rows = db.execute(AGENT_PACKAGE)
    status_ts = {"Done": "endtime", "Start": "starttime"}

    for row in rows:
        if row["Status"] in status_ts:
            result["nodes"][nid][row["Command"]] = result["nodes"][nid].get(row["Command"], {"starttime": "",
                                                                                      "endtime": ""})

            result["nodes"][nid][row["Command"]][status_ts[row["Status"]]] = row[AGENT_TIMESTAMP_FIELD]
            result["nodes"][nid][row["Command"]]["elapsed"] = calculate_elapsed(
                                                result["nodes"][nid][row["Command"]][status_ts["Done"]],
                                                result["nodes"][nid][row["Command"]][status_ts["Start"]])

    gather_agent_raw(result, nid)

    return

def gather_reboot_results(result, nid):
    """ Add reboot entries to results """

    try:
        reboot_list = db.execute(EVENTACTION.format(event_action="Reboot") + \
            "AND (Status='Start' OR Status='Done')")

        start = end = ""
        for reboot_entry in reboot_list:
            if "Start" in reboot_entry["Status"]:
                start = reboot_entry[AGENT_TIMESTAMP_FIELD]
            elif "Done" in reboot_entry["Status"]:
                end = reboot_entry[AGENT_TIMESTAMP_FIELD]

            if start and end:
                elapsed = calculate_elapsed(end, start)

                result["nodes"][nid]["reboots"].append({
                    "starttime": start,
                    "endtime": end,
                    "elapsed": elapsed
                    })
                start = end = ""
    except:
        logger.exception("Error gathering raw data from AgentPersistent-%s" % nid)
        raise

    return

def gather_data(archive_tars, args):
    """ Main driver to gather construct the result """

    def gather_data_helper(archive_tar):

        # Name of tarball
        archive_tar_name = os.path.basename(archive_tar)

        # Name of extracted tar dir; temporary location of extracted tar
        archive_id = os.path.splitext(archive_tar_name)[0]
        archive_extracted_dir = os.path.join(TMP_DIR, archive_id)

        result = {"nodes": {}}
        try:
            tar = tarfile.open(archive_tar_name)
            member_names = tar.getnames()

            upgrade_archive = False

            if "UPGRADE" in str(member_names) and args.upgrade:
                args.upgrade = not args.upgrade if not args.all else args.upgrade
                upgrade_archive = True
                result["Type"] = "UPGRADE"
            elif "PACKAGE" in str(member_names) and args.package:
                args.package = not args.package if not args.all else args.package
                result["Type"] = "PACKAGE"
            elif "FIRMWARE" in str(member_names) and args.firmware:
                args.firmware = not args.firmware if not args.all else args.firmware
                result["Type"] = "FIRMWARE"

            if result.get("Type", None):
                for member_name in member_names:
                    try:
                        extracted_member = None
                        if member_name.endswith(".db"):
                            extracted_member = extract_member_tar(tar, member_name)
                            gather_from_db(result, extracted_member, type_upgrade=upgrade_archive)
                        elif upgrade_archive:
                            if member_name.split("/")[-1].startswith("Agent") and member_name.endswith("log"):
                                extracted_member = extract_member_tar(tar, member_name)
                                gather_from_logfile(result, extracted_member)
                            elif member_name.endswith("UPGRADE"):
                                extracted_member = extract_member_tar(tar, member_name)
                                from_upgrade_flag = {"CurrentVersion": "",
                                                "NextVersion": "", 
                                                "IsRollingUpgrade": ""}
                                with open(extracted_member) as emember_file:
                                    for line in emember_file.readlines():
                                        var, val = line.split("=")
                                        if var in from_upgrade_flag:
                                            from_upgrade_flag[var] = val.strip()
                                result.update(from_upgrade_flag)

                    except:
                        logger.exception("Error extracting or attaching database")
                        raise
                    finally:
                        try:
                            if extracted_member:
                                os.remove(extracted_member)
                        except:
                            logger.error("Error removing %s" % extracted_member)
                            raise

                if args.output:
                    output_result_file(result, args.output, archive_id)
                else:
                    all_results[archive_id] = result

        finally:
            tar.close()
            shutil.rmtree(archive_extracted_dir, ignore_errors=True)

        return

    all_results = {}
    if args.tar:
        gather_data_helper(args.tar)
    else:
        for archive_tar in archive_tars:
            gather_data_helper(archive_tar)

            # Get the latest archive from archive type options specified
            if not args.all and not any((args.upgrade, args.package, args.firmware)):
                break

    if not args.output:
        print(json.dumps(all_results))

    return

def output_result_file(result, output_dir, archive_id):
    result_json = json.dumps(result)

    # Write results to json file
    results_file = os.path.join(output_dir, RESULTS_FILE.format(archiveid=archive_id))
    with open(results_file, "w") as resultsf:
        resultsf.write(result_json)

    return

def get_args():

    parser = optparse.OptionParser(usage="usage: %prog [options] arguments")
    parser.add_option("-a", "--all", help="Run all archives in /ifs/.ifsvar/upgrade/archive/ \n \
        Default: runs for latest archive for archive types given.", action="store_true", default=False)
    parser.add_option("-t", "--tar", help="Tar file", default="")
    parser.add_option("-o", "--output", help="Output Directory. Filename = Archive tar filename", default="")
    parser.add_option("-f", "--firmware", help="Include firmware archive tars", action="store_true", default=False)
    parser.add_option("-p", "--package", help="Include patch archive tars", action="store_true", default=False)
    parser.add_option("-u", "--upgrade", help="Include upgrade archive tars", action="store_true", default=False)
    args = parser.parse_args()[0]

    if args.tar:
        if args.all:
            parser.error("options --all and --file cannot be used together")
        if not os.path.isfile(args.tar):
            raise Failure("File '{0}' does not exist".format(args.tar))
        args.upgrade = args.package = args.firmware = True

    if not args.upgrade and not args.package and not args.firmware:
        parser.error("must specify one or more archive types (--firmware, --package, --upgrade)")

    if args.output and not os.path.isdir(args.output):
        raise Failure("Directory '{0}' does not exist".format(args.output))

    return args

def main():
    try:
        args = get_args()

        if not os.path.isdir(ARCHIVE_DIR):
            raise Failure("Directory '{0}' does not exist".format(ARCHIVE_DIR))

        os.chdir(ARCHIVE_DIR)

        archive_tars = [f for f in os.listdir(".") \
                        if f.startswith("UpgradeArchive") and f.endswith((".tgz", ".tar.gz"))]

        if not archive_tars:
            raise Failure("No upgrade archive tars exist")

        # Sort so that last upgrade archive is first
        archive_tars.sort(reverse=True)

        global db
        db = DATABASE()
        gather_data(archive_tars, args)

    except Failure as ferr:
        logger.error(ferr)
        return 127
    except KeyboardInterrupt:
        print("\nExiting...")
        return 130
    except Exception:
        return 1

if __name__ == "__main__":
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
    logger = logging.getLogger(__name__)

    status = main()
    sys.exit(status)
