#!/usr/local/bin/python

import os
import syslog
import re
import atexit
import sqlite3
import sys
import datetime
import time
from time import mktime
import hashlib

import isi.fs.lin as lin
import isi.fs.enc as enc
import isi.fs.sbt as sbt
import isi.fs.pctl2 as pctl2
import isi.fs.siq as siq
import isi.fs.siq_upgrade as siqup
import isi.celog
from isi.celog.eventids import eventid_map
from sqlite3 import dbapi2 as sqlite
from xml.dom.minidom import parseString, Document

# File locations
ifsvar_tsm_path = "/ifs/.ifsvar/modules/tsm"
config_dir_path = "/ifs/.ifsvar/modules/tsm/config"
upgrade_done_file = "/ifs/.ifsvar/modules/tsm/backup/.ript-upgrade-done"
upgrade_error_file = "/ifs/.ifsvar/modules/tsm/backup/.ript-upgrade-error"
upgrade_lock_file = "/ifs/.ifsvar/modules/tsm/config/.upgrade.lock"
upgrade_lock_fd = -1
path_to_backup = "/ifs/.ifsvar/modules/tsm"
backup_path = "/ifs/.ifsvar/modules/tsm/backup"
old_source_record_file = "/ifs/.ifsvar/modules/tsm/config/source_record.xml"
source_records_path = "/ifs/.ifsvar/modules/tsm/config/source_records/"
source_record_locks_path = "/ifs/.ifsvar/modules/tsm/config/source_record_locks/"
old_target_record_file = "/ifs/.ifsvar/modules/tsm/config/target_record.xml"
target_records_path = "/ifs/.ifsvar/modules/tsm/config/target_records/"
target_record_locks_path = "/ifs/.ifsvar/modules/tsm/config/target_record_locks/"
upgrade_failpoint_file = "/ifs/.ifsvar/modules/tsm/config/.ript-upgrade-failpoint"
alerts_path = "/ifs/.ifsvar/modules/tsm/config/alerts/"
alert_locks_path = "/ifs/.ifsvar/modules/tsm/config/alert_locks/"
reports_db = "/ifs/.ifsvar/modules/tsm/sched/reports/reports.db"
policy_file = "/ifs/.ifsvar/modules/tsm/config/siq-policies.gc"
worker_pool_path = "/ifs/.ifsvar/modules/tsm/config/worker_pool"
config_file = "/ifs/.ifsvar/modules/tsm/config/siq-conf.gc"

# SQL commands
DB_REPORTS_STATUS_EXISTS = ("SELECT name FROM sqlite_master "
                            "WHERE type = 'table' AND name = 'reports_status'")
DB_REPORTS_STATUS_RENAME = ("ALTER TABLE reports_status "
                            "RENAME TO reports_status_tmp")
DB_REPORTS_STATUS_CREATE = ("CREATE TABLE IF NOT EXISTS reports_status ("
                            "policy_id TEXT PRIMARY KEY, "
                            "job_id INTEGER, "
                            "inc_job_id INTEGER)")
DB_REPORTS_STATUS_CREATE_INDICES = ("CREATE INDEX IF NOT EXISTS "
                                    "reports_status_index1 "
                                    "ON reports_status(policy_id)")
DB_REPORTS_STATUS_ADD_FOUND = ("ALTER TABLE reports_status "
                               "ADD COLUMN found INTEGER DEFAULT 0")
DB_REPORTS_STATUS_POPULATE = ("INSERT INTO reports_status "
                              "SELECT policy_id, job_id, inc_job_id "
                              "FROM reports_status_tmp WHERE found = 1")
DB_REPORTS_STATUS_FOUND_PID = ("UPDATE reports_status SET found = 1 "
                               "WHERE policy_id = ?")
DB_REPORTS_STATUS_DROP = ("DROP TABLE reports_status_tmp")
DB_LKG_FOR_POLICY = ("SELECT last_known_good_time FROM reports_status "
                     "WHERE policy_id = ?")

now = datetime.datetime.now()
now_timestamp = int(mktime(now.timetuple()))
backup_file =  "/ifs/.ifsvar/modules/tsm/backup/pre-ript." + str(now_timestamp) + ".tgz"

exit_status = -1

# repstate variables
REP_NAME = "_snap_rep_base_mirrored"
REP_PATH = "repstate"
CPOOLS_NAME = "_cpools_state_base_mirrored"
CPOOLS_PATH = "cpss"

def mksbt(sbtname, cpss_fd, sbt_root_fd):
    # create the SBT
    sbt.sbt_create(sbtname)
    sbt_fd = sbt.sbt_open(sbtname)
    # reprotect
    attr = pctl2.Expattr()
    attr.policy = pctl2.StringToPolicy("default")
    pctl2.set(sbt_fd, attr)
    pctl2.full_restripe(sbt_fd, pctl2.REPROTECT)
    os.close(sbt_fd)
    # mv it to the policy_id/cpss directory
    enc.renameat(sbt_root_fd, sbtname, enc.DEFAULT, cpss_fd, sbtname,
        enc.DEFAULT)

def ck_or_create_cpool_sbt(trec, src_fd, sbt_root_fd):
    trec_fd = rep_fd = cpss_fd = -1
    st = None
    # open the policy sbt directory
    try:
        trec_fd = enc.openat(src_fd, trec.policy_id, enc.DEFAULT,
            os.O_RDONLY)
    except:
        return True
    # open the repstate directory
    try:
        rep_fd = enc.openat(trec_fd, REP_PATH, enc.DEFAULT, os.O_RDONLY)
    except:
        os.close(trec_fd)
        return True
    # look for the base mirrored repstate
    try:
        st = enc.fstatat(rep_fd, "".join((trec.policy_id, REP_NAME)),
            enc.DEFAULT)
    except:
        # if it doesn't exist, we are done
        os.close(trec_fd)
        os.close(rep_fd)
        return True

    # open the cpss directory, create it if missing
    try:
        cpss_fd = enc.openat(trec_fd, CPOOLS_PATH, enc.DEFAULT,
                os.O_RDONLY)
    except:
        try:
            enc.mkdirat(trec_fd, CPOOLS_PATH, enc.DEFAULT, 0770)
            cpss_fd = enc.openat(trec_fd, CPOOLS_PATH, enc.DEFAULT,
                os.O_RDONLY)
        except:
            return False

    # look for the cpool mirrored repstate
    cp_rep_name = "".join((trec.policy_id, CPOOLS_NAME))
    try:
        st = enc.fstatat(cpss_fd, cp_rep_name, enc.DEFAULT)
    except:
        try:
            mksbt(cp_rep_name, cpss_fd, sbt_root_fd)
        except:
            os.close(cpss_fd)
            os.close(rep_fd)
            os.close(trec_fd)
            return False

    os.close(cpss_fd)
    os.close(rep_fd)
    os.close(trec_fd)
    return True

def exiting():
    if exit_status != 0:
        os.system("touch %s" % (upgrade_error_file))
        if (os.access(upgrade_done_file, os.F_OK)):
            os.unlink(upgrade_done_file)
            syslog.syslog(syslog.LOG_NOTICE, "SIQ Upgrade failed.")
        try:
            siq_ev = isi.celog.Event(
                event_id=eventid_map["SW_SIQ_POLICY_UPGRADE"],
                severity=isi.celog.EVENT_LEVEL_CRIT,
                message='SyncIQ upgrade has failed to complete successfully.')
            siq_ev.add()
        except Exception, exc:
            syslog.syslog(syslog.LOG_ERR, "Failed to create upgrade failure alert. Error: %s" % exc)
    else:
        os.system("touch %s" % (upgrade_done_file))
        os.chmod(upgrade_done_file, 0640)
        if (os.access(upgrade_error_file, os.F_OK)):
            os.unlink(upgrade_error_file)
        syslog.syslog(syslog.LOG_NOTICE, "SIQ Upgrade completed.")

    #Release upgrade lock
    if upgrade_lock_fd != -1:
        os.close(upgrade_lock_fd)


atexit.register(exiting)

syslog.openlog("isi_migrate")
syslog.syslog(syslog.LOG_NOTICE, "Starting SyncIQ Upgrade (Ript)...")


if (os.access(upgrade_failpoint_file, os.F_OK)):
    syslog.syslog(syslog.LOG_ERR, " ".join([
        "Detected failpoint file.  Simulating Upgrade Failure. ",
        "To prevent this from happening, remove %s." % upgrade_failpoint_file
    ]))
    sys.exit(-1)

#Grab lock on upgrade file
if (not os.access(ifsvar_tsm_path, os.F_OK)):
    #Nothing exists, we are done
    syslog.syslog(syslog.LOG_INFO, "SIQ config directory not found")
    os.system("mkdir -p %s" % backup_path)
    exit_status = 0
    sys.exit(0)

os.system("mkdir -p %s" % backup_path)
os.system("mkdir -p %s" % config_dir_path)
os.system("mkdir -p %s" % source_records_path)
os.system("mkdir -p %s" % source_record_locks_path)
os.system("mkdir -p %s" % target_records_path)
os.system("mkdir -p %s" % target_record_locks_path)
os.system("mkdir -p %s" % alerts_path)
os.system("mkdir -p %s" % alert_locks_path)
os.system("mkdir -p %s" % worker_pool_path)

upgrade_lock_fd = os.open(upgrade_lock_file, os.O_EXLOCK | os.O_CREAT, 0400)
os.chmod(upgrade_lock_file, 0400)

# BACKUP EXISTING FILES + SANITY CHECK
# Determine if the upgrade needs to happen
if (os.access(upgrade_done_file, os.F_OK)):
    syslog.syslog(syslog.LOG_NOTICE, "Skipping Riptide upgrade.  Upgrade already completed.")
    exit_status = 0
    sys.exit(0)

# Backup old config and sched folders
# Mkdir backup folder
try:
    os.mkdir(backup_path)
except:
    pass

# Make a tarball of old files, but only stuff that exists
to_backup = ""
if (os.access("/ifs/.ifsvar/modules/tsm/config", os.F_OK)):
    to_backup = to_backup + "/ifs/.ifsvar/modules/tsm/config "

if (os.access("/ifs/.ifsvar/modules/tsm/sched/reports", os.F_OK)):
    to_backup = to_backup + "/ifs/.ifsvar/modules/tsm/sched/reports "

if (os.access("/ifs/.ifsvar/modules/tsm/sched/run", os.F_OK)):
    to_backup = to_backup + "/ifs/.ifsvar/modules/tsm/sched/run "

if (os.access("/etc/mcp/override/tsm.xml", os.F_OK)):
    to_backup = to_backup + "/etc/mcp/override/tsm.xml "

if (os.access("/etc/mcp/override/bwt.xml", os.F_OK)):
    to_backup = to_backup + "/etc/mcp/override/bwt.xml "

if (os.access("/etc/mcp/sys/tsm.xml", os.F_OK)):
    to_backup = to_backup + "/etc/mcp/sys/tsm.xml "

tarcmd = "tar -c -z -f %s %s > /dev/null 2>&1" % (backup_file, to_backup)
syslog.syslog(syslog.LOG_INFO, "Executing tar: \"%s\"" % (tarcmd))
os.system(tarcmd)
os.chmod(backup_file, 0640)

def iterate_and_save_source_records(source_recs, cursor):
    for srec in source_recs:
        pid = srec.attributes["policy-id"].value

        lkg = 0
        if cursor:
            try:
                cursor.execute(DB_LKG_FOR_POLICY, (pid,))
                res = cursor.fetchone()
                if res:
                    lkg = res[0]
                cursor.execute(DB_REPORTS_STATUS_FOUND_PID, (pid,));
            except sqlite3.OperationalError:
                pass

        name = source_records_path + pid + ".xml"
        fd = os.open(name, os.O_CREAT | os.O_RDWR, 0640)
        f = os.fdopen(fd, "w+")
        doc = Document()
        base = doc.createElement("siq-source")
        doc.appendChild(base)

        newline = doc.createTextNode("\n  ")
        records = doc.createElement("policy-records")
        base.appendChild(newline)
        base.appendChild(records)
        newline = doc.createTextNode("\n")
        base.appendChild(newline)

        newline = doc.createTextNode("\n    ")
        records.appendChild(newline)

        space = doc.createTextNode("  ")
        srec.appendChild(space)
        job_delay_start = doc.createElement("job-delay-start")
        job_delay_start_value = doc.createTextNode("0")
        job_delay_start.appendChild(job_delay_start_value)
        srec.appendChild(job_delay_start)
        newline = doc.createTextNode("\n    ")
        srec.appendChild(newline)

        space = doc.createTextNode("  ")
        srec.appendChild(space)
        lkg_element = doc.createElement("last-known-good-time")
        lkg_value = doc.createTextNode(str(lkg))
        lkg_element.appendChild(lkg_value)
        srec.appendChild(lkg_element)
        newline = doc.createTextNode("\n    ")
        srec.appendChild(newline)

        # Set job versions in existing source records to the most recent
        # pre-Riptide value to ensure version continuity for failed/paused
        # jobs.

        fields = ["composite-job-ver-local",
            "composite-job-ver-common",
            "prev-composite-job-ver-local",
            "prev-composite-job-ver-common",
            "restore-composite-job-ver-local",
            "restore-composite-job-ver-common",
            "restore-prev-composite-job-ver-local",
            "restore-prev-composite-job-ver-common"]

        for field in fields:
            space = doc.createTextNode("  ")
            srec.appendChild(space)
            element = doc.createElement(field)
            value = doc.createTextNode("%u" % siq.MSG_VERSION_CHOPUVILLE)
            element.appendChild(value)
            srec.appendChild(element)
            newline = doc.createTextNode("\n    ")
            srec.appendChild(newline)

        records.appendChild(srec)
        newline = doc.createTextNode("\n  ")
        records.appendChild(newline)

        f.write(doc.toxml(encoding='utf-8'))
        f.close()

# UPGRADE SOURCE RECORDS
src_file = None
src_dom = None
print "Upgrading Source Records..."

try:
    conn = None
    cursor = None
    if (os.access(old_source_record_file, os.F_OK)):
        src_file, src_dom = siqup.open_and_parse_xml(old_source_record_file)
        nodes = src_dom.getElementsByTagName("policy-record")
        if (len(nodes) >= 1):
            try:
                conn = sqlite3.connect(reports_db)
                cursor = conn.cursor()
                cursor.execute(DB_REPORTS_STATUS_EXISTS)
                if cursor.fetchone():
                    cursor.execute(DB_REPORTS_STATUS_ADD_FOUND)
                else:
                    # Table doesn't exist
                    cursor = None
            except sqlite3.OperationalError, exc:
                syslog.syslog(syslog.LOG_ERR,
                              "Unable to access records db. Error: %s" % exc)
            iterate_and_save_source_records(nodes, cursor)

            # Remake the reports_status db without the lkg column
            if cursor:
                cursor.execute(DB_REPORTS_STATUS_RENAME)
                cursor.execute(DB_REPORTS_STATUS_CREATE)
                cursor.execute(DB_REPORTS_STATUS_CREATE_INDICES)
                cursor.execute(DB_REPORTS_STATUS_POPULATE)
                cursor.execute(DB_REPORTS_STATUS_DROP)
                conn.commit()
    else:
        syslog.syslog(syslog.LOG_INFO, "Skipping source record.  No source record found.")
except Exception, exc:
        syslog.syslog(syslog.LOG_ERR, "Failed to upgrade source record.  Error: %s" %(exc))
        sys.exit(-1)
finally:
    if conn:
        conn.close()
    if src_file:
        src_file.close()
    if (os.access(old_source_record_file, os.F_OK)):
        os.unlink(old_source_record_file)

def iterate_and_save_target_records(tgt_recs):
    for trec in tgt_recs:
        pid_nodes = trec.getElementsByTagName("id")
        pid = pid_nodes[0].firstChild.data

        # 5.0 target records have a pseudo policy id with special characters
        # in it. Rather than creating a file with special characters in it,
        # we'll hash the pseudo id and name the file using that. The synciq
        # target record library will use the same hash algorithm to locate
        # these files.
        if '.' in pid:
            pid = hashlib.md5(pid).hexdigest()


        name = target_records_path + pid + ".xml"
        fd = os.open(name, os.O_CREAT | os.O_RDWR, 0640)
        f = os.fdopen(fd, "w+")
        doc = Document()
        base = doc.createElement("target-status")
        doc.appendChild(base)

        newline = doc.createTextNode("\n  ")
        base.appendChild(newline)

        # Set job versions in existing target records to the most recent
        # pre-Riptide value to ensure version continuity for failed/paused
        # jobs.

        fields = ["composite-job-ver-local", "composite-job-ver-common",
            "prev-composite-job-ver-local", "prev-composite-job-ver-common"]

        for field in fields:
            space = doc.createTextNode("  ")
            trec.appendChild(space)
            element = doc.createElement(field)
            value = doc.createTextNode("%u" % siq.MSG_VERSION_CHOPUVILLE)
            element.appendChild(value)
            trec.appendChild(element)
            newline = doc.createTextNode("\n  ")
            trec.appendChild(newline)

        base.appendChild(trec)
        newline = doc.createTextNode("\n")
        base.appendChild(newline)

        f.write(doc.toxml(encoding='utf-8'))
        f.close()

# UPGRADE TARGET RECORDS
tgt_file = None
tgt_dom = None
print "Upgrading Target Records..."

try:
    if (os.access(old_target_record_file, os.F_OK)):
        tgt_file, tgt_dom = siqup.open_and_parse_xml(old_target_record_file)
        nodes = tgt_dom.getElementsByTagName("policy")
        if (len(nodes) >= 1):
            iterate_and_save_target_records(nodes)
    else:
        syslog.syslog(syslog.LOG_INFO, "Skipping target record.  No target record found.")
except Exception, exc:
        syslog.syslog(syslog.LOG_ERR, "Failed to upgrade target record.  Error: %s" %(exc))
        sys.exit(-1)
finally:
    if tgt_file:
        tgt_file.close()
    if (os.access(old_target_record_file, os.F_OK)):
        os.unlink(old_target_record_file)

def iterate_and_save_policies():
    synciq = siq.SyncIQ()
    siq.lock_policies()
    policies = synciq.get_policies()
    now = int(time.time())
    for policy in policies.values():
        policyconfig = policy.get_policy_config()
        if policyconfig.creation_time == 0:
            policyconfig.creation_time = now
            try:
                policyconfig.save(True)
            except Exception, exc:
                syslog.syslog(syslog.LOG_ERR, "Failed to upgrade policy %s."
                        % policy.name)
                siq.unlock_policies()
                raise
    # create cpools repstates
    trecs = synciq.get_target_status()
    if len(trecs) > 0:
        sbt_root_fd = src_fd = -1
        try:
            sbt_root_fd = lin.open((5 ,lin.HEAD_SNAPID), os.O_RDONLY)
        except:
            sbt_root_fd = -1
            syslog.syslog(syslog.LOG_ERR, "Failed to open the SBT LIN")
            siq.unlock_policies()
            raise

        if sbt_root_fd != -1:
            # if there isn't already a 5/synciq/src directory, just
            # continue
            try:
                src_fd = enc.openat(sbt_root_fd,
                    os.path.join("synciq","src"),
                    enc.DEFAULT, os.O_RDONLY)
            except:
                src_fd = -1

            if src_fd != -1:
                trec_fd = rep_fd = cpss_fd = -1
                for trec in trecs:
                    if not ck_or_create_cpool_sbt(trec, src_fd, sbt_root_fd):
                        syslog.syslog(syslog.LOG_ERR, " ".join((
                            "Failed to create CloudPools mirror repstate",
                            "for policy ID", trec.policy_id)))

                os.close(src_fd)

            os.close(sbt_root_fd)
    siq.unlock_policies()

# UPGRADE POLICIES
syslog.syslog(syslog.LOG_INFO, "Upgrading policies...")
print "Upgrading policies..."

try:
    if (os.access(policy_file, os.F_OK)):
        iterate_and_save_policies()
    else:
        syslog.syslog(syslog.LOG_INFO, "Skipping policies. No policies found.")
except Exception, exc:
    syslog.syslog(syslog.LOG_ERR, "Failed to upgrade policies. Error: %s" % (exc))
    sys.exit(-1)

# UPGRADE CONFIG
syslog.syslog(syslog.LOG_INFO, "Upgrading siq global config...")
print "Upgrading siq global config..."

try:
    if (os.access(config_file, os.F_OK)):
        synciq = siq.SyncIQ()
        synciq.max_concurrent_jobs = 50
        synciq.save(True)
    else:
        syslog.syslog(syslog.LOG_INFO, "Skipping config update. No config found.")
except Exception, exc:
    syslog.syslog(syslog.LOG_ERR, "Failed to upgrade config. Error: %s" % (exc))
    sys.exit(-1)

# END OF UPGRADE - Set status for exiting function
exit_status = 0
