#!/usr/bin/env python

# Copyright C 2002-2010 by Isilon Systems, Inc. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
#
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
#
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems.
#
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.

import os
import re
import signal
import sys
import syslog
import time

import isi.app.lib.procs as procs
import isi.config.arrcfg as arrcfg
import isi.config.rc as rclib
import isi.error as ierr
import isi.fs.journal as jlib
import isi.gconfig as gconf
import isi.hw.fcb as fcb
import isi.hw.bootstrap as bootstrap
import isi.hw.hal as hal
import isi.hw.ipmi as ipmi
import isi.hw.pmp as pmp
import isi.hw.version as hwver
import isi.misc
import isi.ui.lib.format as fmt


def init_syslog():
    syslog.openlog(sys.argv[0])

def clean_syslog():
    syslog.closelog()

def log_print(msg):
    syslog.syslog(syslog.LOG_NOTICE, msg)
    print msg

def kill_other(pid):
    try:
        os.kill(pid, signal.SIGINT)
    except os.error:
        pass

def stop_boot(msg):
    print msg
    # This allows isi_testjournal_rc to fail out and handle stop_boot
    sys.exit(1)

def is_newjournaled():
    ctx = gconf.open(gconf.gcfg_nodestatus).ctx_new()
    return ctx.read("journal_info.newjournaled_without_mount")

def first_post_ifs_ran():
    return os.path.exists("/etc/ifs/isi_first_post_ifs_already_ran")

def disk_restore_to_dram(buddy=False):
    restore_arg = ''
    if buddy:
        restore_arg = ' -b'
    if os.system('/usr/sbin/isi_restore_journal%s' % restore_arg) == 0:
        jlib.clear_bad()
        return True
    else:
        return False

def dram_new_journal():
    os.system('/usr/sbin/isi_newjournal --force -f %s' %
            hal.getJournalDevicePath())
    jlib.clear_bad()

def dram_save_to_disk():
    return os.system("/usr/sbin/isi_save_journal") == 0

def recover_journal(restoreFromFlash):

    # If there hasn't been a mount since the journal was newjournal'd,
    # try reinitializing the journal to clear any ECC errors and try
    # the validation again.
    if is_newjournaled():
        log_print("Attempting to reinitialize journal...")
        dram_new_journal()
        if jlib.dram_valid():
            log_print("Reinitialize succeeded")
            return True

    if restoreFromFlash:
        log_print("Reinitialize failed")

        # Prefer flash copy over disk backup

        if not hal.hasNvramAutoRestore():
            # The hardware does not support autorestore.
            # Attempt to do a manual restore from flash.
            if jlib.flash_valid():
                log_print("Attempting to restore journal from flash backup...")
                exc_str = None
                try:
                    jlib.flash_restore_to_dram()
                    if jlib.dram_valid():
                        log_print("Restore from flash succeeded")
                        return True
                except ierr.IsiError, e:
                    exc_str = ": encountered error: %s" % str(e)
                    log_print("Restore from flash failed%s" %
                              exc_str if exc_str is not None else "")
            else:
                if jlib.dram_ready():
                    try:
                        if jlib.dram_valid():
                            log_print("Restore from flash succeeded")
                            return True
                    except ierr.IsiError, e:
                        exc_str = ": encountered error: %s" % str(e)
                        log_print("Restore from flash failed%s" %
                                    exc_str if exc_str is not None else "")

        log_print("Restore from flash failed")

    if jlib.disk_valid():
        log_print("Attempting to restore journal from disk backup...")
        if disk_restore_to_dram():
            if jlib.dram_valid():
                log_print("Restore from disk succeeded")
                return True

    log_print("Restore from disk failed")

    log_print("Attempting to save and restore journal to clear any "
              "ECC errors in unused DRAM blocks...")

    if dram_save_to_disk():
        log_print("Save succeeded, restoring...")
        if disk_restore_to_dram():
            if jlib.dram_valid():
                log_print("Restore succeeded")
                return True

    log_print("Restore failed")
    return False

def zero_journal(buddy=False):
    if buddy:
        journal_dev = hal.getJournalPeerDevicePath()
    else:
        journal_dev = hal.getJournalDevicePath()
    cmd = '/bin/dd if=/dev/zero of=%s bs=512 count=1'
    error, output = procs.get_cmd_output(cmd)
    if error:
        log_print("Unable to zero out journal superblock (%s)" % journal_dev)

def check_buddy_clustered():
    result = False
    acfg = arrcfg.config_load()
    aguid = acfg.guid
    peer_sn = hal.getPeerSerialNumber()
    if peer_sn is not None:
        adevid = acfg.find_device_by_serial_number(peer_sn)
        # If we can't find our buddy device in the array config, assume
        # we're not clustered with it
        if adevid is not None:
            result = True
        else:
            log_print("Local node is not clustered with peer node.")
    return result

def recover_mirror_journal():
    recover_objs = [
        # Local copy
        {
        'dram_valid' : jlib.dram_valid,
        'disk_valid' : jlib.disk_valid,
        'journal_ident' : 'local',
        'buddy' : False,
        },
        # Buddy copy
        {
        'dram_valid' : jlib.dram_buddy_valid,
        'disk_valid' : jlib.disk_buddy_valid,
        'journal_ident' : 'peer',
        'buddy' : True,
        }
    ]

    # If the node is not configured, only check local
    if not arrcfg.is_local_configured() or not check_buddy_clustered():
        recover_objs.pop()

    for obj in recover_objs:
        skip_check = False
        journal_ident = obj.get('journal_ident')
        buddy = obj.get('buddy')
        dram_valid_func = obj.get('dram_valid')
        disk_valid_func = obj.get('disk_valid')
        if dram_valid_func():
            pass
        elif disk_valid_func():
            # If DRAM is invalid, but disk backup is valid, restore backup
            log_print("Attempting to restore %s journal from disk backup." %
                journal_ident)
            if disk_restore_to_dram(buddy):
                if not dram_valid_func():
                    log_print("Unable to restore %s journal from disk"
                        " backup. Zero out superblock." % journal_ident)
                    zero_journal(buddy)
                    skip_check = True
        elif not buddy and not dram_valid_func() and not disk_valid_func():
            if is_newjournaled():
                log_print("Attempting to reinitialize %s journal..." %
                    journal_ident)
                dram_new_journal()
                if jlib.dram_valid():
                    log_print("Reinitialize succeeded")
                else:
                    # Something's very wrong here if dram_valid()
                    # is not true and is_newjournaled() is True,
                    # so we stop boot.
                    stop_boot("Unable to reinitialize %s journal. "
                        "Contact EMC Customer Support immediately" %
                        journal_ident)
            else:
                log_print("DRAM and disk backup are invalid for %s journal. "
                    "Zero out journal superblock." % journal_ident)
                zero_journal(buddy)
                skip_check = True
        else:
            # If neither DRAM or disk backup are valid, zero out the
            # superblock
            log_print("DRAM and disk backup are invalid for %s journal. "
                "Zero out journal superblock." % journal_ident)
            zero_journal(buddy)
            skip_check = True
        if not skip_check:
            check_journal_version(buddy, True, False)

# check_journal_version()
# buddy      - Whether or not we're checking local or buddy node.
# mirror     - If mirrored journal is being run on this system.
#              XXX PSI PIPELINE this info should probably come from
#              PSI
# exit_early - If a failure triggers stop_boot()
def check_journal_version(buddy=False, mirror=False, exit_early=True):
    # Check the journal version. If we have a newer journal
    # than the code knows about, something's horribly
    # wrong and we need to stop boot.
    version_check_pass = True
    ident = ''
    if mirror:
        if buddy:
            ident = 'Peer '
        else:
            ident = 'Local '

    jver = jlib.get_version(buddy)
    if not jlib.version_supported(jver):
        log_print("Unsupported journal version %d" % jver)
        msg = ("This node has an unsupported version of the %sjournal: "
                "v%d. Contact EMC Customer Support immediately." % (ident, jver))
        if not exit_early:
            log_print(msg)
            return
        else:
            stop_boot(msg)

    try:
        (jguid, jdevid) = jlib.get_guid_and_devid(buddy)
        if long(jguid, 16) == 0L and jdevid == 0:
            if buddy:
                # We can't confirm if the buddy has been newjournal'ed, but we
                # can assume it has been if the cluster GUID and devid are 0.
                # If this is the case, log and return.
                log_print("%sjournal appears to have a new journal." % ident)
                return
            elif is_newjournaled():
                # For local, we can check node_status.gc for whether or not
                # we newjournal'ed when mounted.
                #
                # If isi_post_ifs has already ran and we have a new journal,
                # stop boot.
                if first_post_ifs_ran():
                    stop_boot("This node belongs to a cluster but appears to "
                            "have a new journal. If this is expected, you may "
                            "continue booting. If this is NOT expected or you "
                            "are not sure, DO NOT continue booting and contact "
                            "EMC Customer Support immediately.")
                else:
                    log_print("Journal new")
                if not exit_early:
                    return
                else:
                    sys.exit(0)

        acfg = arrcfg.config_load()
        aguid = acfg.guid
        adevid = acfg.local_devid
        if buddy:
            adevid = acfg.find_device_by_serial_number(
                    hal.getPeerSerialNumber())
            if adevid is None:
                adevid = -1
            else:
                adevid = adevid.devid

        if ((jguid != aguid) or (jdevid != adevid)):
            # About the only way to get here is messing up disk tango.
            # The journal is self-consistent, but doesn't match the node.
            msg = ("The %sjournal state is internally consistent, but "
                    "the %sjournal state does not match the node and cluster "
                    "state. Journal: cluster guid %s, device id %d. Disk: "
                    "cluster guid %s, device id %d."
                    % (ident, ident, jguid, jdevid, aguid, adevid))
            if not exit_early:
                log_print(msg)
                version_check_pass = False
            else:
                stop_boot("%s Contact EMC Customer Support immediately." % msg)
    except Exception, e:
        # We don't expect any of this to fail so if it does, stop the boot
        # and display the error.
        stop_boot("Internal error occurred while comparing node and "
                "journal identifiers: %s. Contact EMC Customer Support "
                "immediately" % str(e))

    if version_check_pass:
        log_print("%sJournal good" % ident)

def log_journal_info():
    # Print and log NVRAM SN and PN if they exist
    ctx = gconf.open(hal.gcfg_probeconfig).ctx_new()
    pcs = ctx.read("probeconfigs")
    for pc in pcs:
        if "nvram" in pc["type"]:
            (error, output) = procs.get_cmd_output(pc["probe_cmd"])
            if not error and len(output) > 0:
                cmd = "%s | %s" % (pc["vendorid_cmd"], pc["vendorid_filter"])
                (error, output) = procs.get_cmd_output(cmd)
                if not error and len(output) > 0:
                    log_print("NVRAM PN: %s" % output[0])
                cmd = "%s | %s" % (pc["serialno_cmd"], pc["serialno_filter"])
                (error, output) = procs.get_cmd_output(cmd)
                if not error and len(output) > 0:
                    log_print("NVRAM SN: %s" % output[0])

def reboot_in_progress():
    result = False
    sensor = None
    if bootstrap.isInfinityMotherboard():
        sensor_str = "Shutdown_In_Prog"
        sensors = hal.getSensors()
        if sensors is not None:
            sensor = sensors.get(sensor_str)
        if sensor is not None:
            state = sensor.get('state')
            try:
                state = int(state, 16)
                state = ((state & 0x00FF) << 8) |  ((state & 0xFF00) >> 8)
                # Phobos Sensor Functional Spec Table 29
                # Check for reboot in progress
                result = ((state & 0x2) >> 1) and True or False
            except (TypeError, ValueError):
                log_print("Unable to get state from %s sensor" % sensor_str)
                result = True
        else:
            log_print("Unable to read %s sensor" % sensor_str)
            result = True
    return result

def battery_ready_for_holdup():
    result = True
    sensor = None
    if bootstrap.isInfinityMotherboard():
        sensor_str = "Battery0_I2C"
        sensors = hal.getSensors()
        if sensors is not None:
            sensor = sensors.get(sensor_str)
        if sensor is not None:
            state = sensor.get('state')
            try:
                state = int(state, 16)
                state = ((state & 0x00FF) << 8) |  ((state & 0xFF00) >> 8)
                # Phobos Sensor Functional Spec Table 29
                # Check for the ready bit (Ready for Hold Up) being asserted
                result = ((state & 0x10) >> 4) and True or False
            except (TypeError, ValueError):
                log_print("Unable to get state from %s sensor" % sensor_str)
                result = False
        else:
            log_print("Unable to read %s sensor" % sensor_str)
            result = False
    return result

def battery_good(batt):
    # As part of checking for whether or not the battery is good,
    # for Infinity we want to check for BBU ready. If BBU ready
    # is asserted, we'll want to arm. Not arming after BBU ready
    # results in always getting a non-green battery status
    if bootstrap.isInfinityMotherboard():
        if not battery_ready_for_holdup():
            return False
        else:
            try:
                jlib.nvr_arm_flash()
            except ierr.IsiSystemError:
                stop_boot("Unable to arm for vault.")
    status = hal.nvramBatteryStatus(batt)
    if status is None:
        return False
    return hal.nvramBatteryStatusColor(status) == fcb.BR_GREEN

def main():
    # Exit immediately if we're running on a node with no NVRAM.
    if not hal.isClass(hal.STORAGE_CLASS):
        sys.exit(0)

    # XXX PSI PIPELINE - We're assuming that we support PMP if
    # this is an Infinity node.
    if bootstrap.isInfinityMotherboard():
        if pmp.sendJournalRestoreState() is None:
            log_print("Error sending journal restore state to driver")

    # Exit immediately if we're running from a manufacturing CF.
    rc_hash = rclib.parse()
    if rc_hash.has_key("manufacturing") and \
            re.search("[Yy][Ee][Ss]", rc_hash["manufacturing"]) != None:
        sys.exit(0)

    #
    # When a node comes from manufacturing, it should be in "ship mode".
    # This state keeps the batteries from being totally drained while the
    # node is on the shelf. Check to see if the node is in ship mode. If it
    # is, turn it off. This will allow the batteries to get fully charged
    # and support the nvram.
    #
    skipAutoRestore = False
    if hal.hasBatteryShipMode():
        shipMode = hal.nvramBatteryShipModeGet()
        if shipMode is None:
            log_print("Unable to get battery ship mode.")
        elif shipMode == 1:
            hal.nvramBatteryShipModeSet(0)
            shipmode = hal.nvramBatteryShipModeGet()
            if shipmode:
                stop_boot("Verifying ship mode: Batteries have NOT been taken"
                          " out of ship mode.")
            elif shipmode is None:
                stop_boot("Verifying ship mode: Unable to get battery ship mode")
            else:
                log_print("Verifying ship mode: Batteries have been taken out"
                          " of ship mode.")
            # do not attempt a journal restore/recovery on first boot (LOx)
            skipAutoRestore = True

    log_print("Checking Battery Health...")
    numbat = hal.getNumBatteries()
    batteries = range(1, numbat + 1)
    # Bug 170597
    # It takes time (about 10 seconds) from when we turn off ship mode until
    # we get back good battery states, so we can't trust the states immediately
    # after ship mode was disabled.
    batt_good = False
    retry = 20
    while retry > 0:
        batt_bad = [batt for batt in batteries if not battery_good(batt)]
        # LOx only requires one healthy battery to stay in RW mode,
        # whereas previous platforms require that two batteries be
        # healthy to remain in RW.
        if hal.getNvramType() == hwver.NVR_LOX:
            batt_good = len(batt_bad) <= 1
        else:
            batt_good = len(batt_bad) == 0
        if batt_good:
            break
        time.sleep(1)
        retry -= 1
    continue_boot = True
    if not batt_good:
        continue_boot = False
    if len(batt_bad) > 0:
        batt_bad = [str(batt) for batt in batt_bad]
        if len(batt_bad) > 1:
            batt_str = "Batteries %s are unhealthy." % " and ".join(batt_bad)
        else:
            batt_str = "Battery %s is unhealthy." % batt_bad[0]
    else:
        batt_str = "Batteries good"
    if not continue_boot and not arrcfg.is_local_configured():
        stop_boot("%s Check that the batteries are fully charged, then reboot. "
                  "If the batteries are fully charged and not healthy contact "
                  "EMC Customer Support for replacements." % batt_str)
    else:
        log_print(batt_str)

    # RFD XXX - Check for whether or not the node is in a delayed reboot,
    # if so, stop boot. This is only pertinent to Infinity
    retry = 20
    while retry > 0:
        if reboot_in_progress():
            time.sleep(1)
            retry -= 1
        else:
            break
    # Exhausted retries
    if not retry:
        stop_boot("Node is currently in a delayed reboot. Stopping boot to protect "
                  "journal.")
    log_print("Checking Isilon Journal integrity...")
    sys.stdout.flush()

    # If /ifs is mounted, just to the basic battery, block, and ECC tests;
    # do not actually scan/parse (i.e. walk) the journal data.
    if isi.misc.ifs_mounted():
        if jlib.dram_valid(do_walk=False):
            log_print("Mounted DRAM journal check: good")
        else:
            log_print("Mounted DRAM journal check: BAD")

        sys.exit(0)

    devFile = hal.getJournalDevicePath()
    if devFile is None or not os.path.exists(devFile):
        stop_boot("The node does not have the expected NVRAM device %s. "
                  "Contact EMC Customer Support immediately." % devFile)

    # XXX PSI PIPELINE- Check for Chassis Type for now, but we may want a better
    # check from HAL such as hal.supportsMirrorJournal().
    if hal.getChassisType(0) == hwver.CHASSIS_INFINITY:
        # Set the journal bad 'flag' to make the other journal utilities
        # behave correctly.
        jlib.mark_bad()
        skip_checks = recover_mirror_journal()

    # For Belknap and Rocketfuel journals
    # At initial boot, when /ifs isn't mounted yet, we do a full journal
    # walk. For safety, we should never trust an auxiliary file to tell us
    # the journal is bad. If we have a valid journal in NVRAM, we should
    # never overwrite it. If a user needs to force an overwrite for some
    # reason, they can always use dd to zero the journal superblock
    # (the first 512 bytes).
    #
    # For LOX:
    # After a clean shutdown or reboot, don't expect journal in NVRAM. This is
    # special case for NVRAM cards that clear NVRAM content on a graceful
    # shutdown or reboot. For this type of board, the only situations in
    # which NVRAM has a valid journal is when a vault restore has
    # successfully occurred, or after a sudden reset.
    #
    # If we are coming from a graceful shutdown or reboot, the journal will
    # be recovered from the on disk journal backup.
    elif hal.hasNvramAutoRestore() and not skipAutoRestore:
        restore_status = {
            hal.HAL_NVRAM_RESTORE_STATUS_NOT_SUPPORTED : "Unsupported",
            hal.HAL_NVRAM_RESTORE_STATUS_NONE : "Not performed",
            hal.HAL_NVRAM_RESTORE_STATUS_SUCCESS : "Succeeded",
            hal.HAL_NVRAM_RESTORE_STATUS_FAILURE : "Failed"
            }
        log_print("NVRAM autorestore status: %s..." %
            restore_status.get(hal.nvramRestoreStatus(), "Unknown"))

        # Before we check dram below, let's first check for a valid
        # disk backup. For LOX, on a clean shutdown or reboot, we save
        # a journal backup and allow for dram to be invalidated, so we do
        # not want to check the dram before it makes sense to.
        #
        # The existing journal libraries within the dram_valid call below
        # will throw errors for invalid superblock if we scan the dram right
        # away, which could cause confusion/concern in the field unnecessarily.
        if jlib.disk_valid():
            # Set the journal bad 'flag' to make the other journal utilities
            # behave correctly.
            jlib.mark_bad()

            if not recover_journal(False):
                stop_boot("Could not recover journal. Contact Isilon "
                        "Customer Support immediately.")
            else:
                log_print("DRAM journal recovery succeeded")
        elif not jlib.dram_valid():
                stop_boot("Could not restore journal. Contact Isilon "
                        "Customer Support immediately.")
        check_journal_version()
    elif not jlib.dram_valid():
        # Set the journal bad 'flag' to make the other journal utilities
        # behave correctly.
        jlib.mark_bad()

        log_print("DRAM journal is invalid, initiating recovery...")

        if not recover_journal(True):
            stop_boot("Could not recover journal. Contact Isilon "
                      "Customer Support immediately.")
        else:
            log_print("DRAM journal recovery succeeded")
        check_journal_version()

    jlib.clear_bad()

    log_journal_info()
    sys.exit(0)

if __name__ == "__main__":
    init_syslog()
    ipmi.disableCacheReads()
    try:
        main()
    finally:
        clean_syslog()
