#!/usr/libexec/isilon/isi_drivemon

import isi.app.lib.cluster
import isi.app.daemon as daemon
import isi.app.lib.process as process
import isi.app.lib.procs as procs
import isi.celog.eventids as eventids
import isi.celog.query
import isi.config.arrcfg as arrcfg
import isi.hw.bay as drivebay
import isi.hw.drived as drived
import isi.hw.drivemon as libdrivemon
import isi.hw.bootstrap as bootstrap
import isi.hw.hal as hal
import isi.hw.drive_config as drive_config
import inspect
import optparse
import os 
import signal
import sys
import syslog
import time
import traceback

DRIVEMON_ID = eventids.eventid_map["SYS_DRIVEMON"]
MISSING_CONFIG_ID = eventids.eventid_map["SYS_DRIVEMON_MISSING_CONFIG"]
SMART_STATUS_ID = eventids.eventid_map["SYS_DRIVEMON_SMARTSTATUS"]
ADD_DELAY = 0.2


RADISH_LOG_FILE	= "/var/log/isi_radish.log"
BOOTDISK_LOG_FILE = "/var/log/isi_radish_bootdisk.log"
JOURNAL_LOG_FILE = "/var/log/isi_radish_journal.log"
DA_LOG_FILE = "/var/log/hw_da.log"
ATA_LOG_FILE = "/var/log/hw_ata.log"

# 86400 seconds
DEFAULT_SLEEP = (24 * 60 * 60)

WAIT_TIME = 60
WAIT_SLEEP = 10
WAIT_COUNT = (WAIT_TIME / WAIT_SLEEP)

GET_DB_WAIT = 10
GET_DB_MAX_ATTEMPTS = 6

MAX_CANCEL_ATTEMPTS = 10
CANCEL_DELAY = 0.1

class DriveClass(object):
    """
    As of Infinity we are also monitoring the journal drive.
    This class will be used as an enum to do comparisons instead
    of passing around booleans for bootdisk like preInfinity
    drivemon. I am hoping this will be more maintainable in
    the future this way as well.
    """
    DATA = 0
    JOURNAL = 1
    BOOT = 2

INFINITY = bootstrap.isInfinityMotherboard()
READ_ONLY_SCRIPT = "/usr/bin/isi_hwtools/isi_read_only"
JOURNAL_RO_BIT = "system-journal-wear"

drive_configs = {}
model_list = []

mfginfo_list = []
boot_bay = []
journal_bays = []

sighup_count = 1
terminate = False

celog_event_cache = {}

CELOG_SEVERITIES = {
    isi.celog.EVENT_LEVEL_CRIT : 'Critical',
    isi.celog.EVENT_LEVEL_WARN : 'Warning',
    }

WARN_ACTION_MSG = " Please schedule drive replacement."
CRIT_ACTION_MSG = " Please schedule drive replacement immediately."
SMARTFAIL_ACTION_MSG = (" Drive is being removed from the system -"
                        " please schedule drive replacement immediately.")

CallerLine = lambda: inspect.stack()[2][2] 
CallerName = lambda: inspect.stack()[2][3]

def unhandled_exception():
    etype, evalue, etb = sys.exc_info()
    syslog.syslog(syslog.LOG_ERR,
                    "Exception caught during %s (line %s): %s, %s\n" %
		    (CallerName(), CallerLine(), etype, evalue))
    # syslog.syslog(syslog.LOG_INFO,
    #                ("Caller: %s:%s\n" % (CallerName(), CallerLine())))
    lines = traceback.format_exception(etype, evalue, etb)
    lines = reduce(lambda a,b: a + b, map(lambda s: s.splitlines(), lines))
    for line in lines:
        syslog.syslog(syslog.LOG_INFO, "%s\n" % line)


def signal_handler_sigterm(signum, frame):
    global terminate
    syslog.syslog(syslog.LOG_INFO, "Received SIGTERM")
    terminate = True

def signal_handler_sigint(signum, frame):
    global terminate
    syslog.syslog(syslog.LOG_INFO, "Received SIGINT")
    terminate = True

def signal_handler_sighup(signum, frame):
    global received_sighup
    global sighup_count
    received_sighup = True
    syslog.syslog(syslog.LOG_INFO, "Received SIGHUP #%d" % sighup_count)
    sighup_count = sighup_count + 1

def log_boot_drive_info(log_string):
    if not INFINITY:
        command = '(echo \"%s\"; date) >> %s' % (log_string, ATA_LOG_FILE)
        os.system(command)

        command = 'sysctl kern.cam.ada | grep -v "use -D" | grep -v inject_ >> %s' % ATA_LOG_FILE
        os.system(command)

        bootdisks = bootstrap.getEligibleBootdiskDeviceNames()
    else:
        command = '(echo \"%s\"; date) >> %s' % (log_string, DA_LOG_FILE)
        os.system(command)

        # Log the global da settings.
        command = 'sysctl kern.cam.da | grep -v "use -D" | grep -v inject_ | grep -v kern.cam.da.\[0-9\] >> %s' % DA_LOG_FILE
        os.system(command)

        # grab the boot disks only.
        bootdisks = bootstrap.getBootdiskDeviceNames()
        for bootdisk in bootdisks:
            command = 'sysctl kern.cam.da | grep -v "use -D" | grep -v inject_ | grep kern.cam.da.%s >> %s' % (bootdisk[2:], DA_LOG_FILE)
            os.system(command)

    command = '(echo \"%s\"; date) >> %s' % (log_string, BOOTDISK_LOG_FILE)
    os.system(command)

    for bootdisk in bootdisks:
        command = 'isi_radish -qt %s >> %s' % (bootdisk, BOOTDISK_LOG_FILE)
        os.system(command)

# Only used on Infinity Platforms.
def log_journal_drive_info(log_string):
    command = '(echo \"%s\"; date) >> %s' % (log_string, ATA_LOG_FILE)
    os.system(command)

    command = 'sysctl kern.cam.ada | grep -v "use -D" | grep -v inject_ >> %s' % ATA_LOG_FILE
    os.system(command)

    command = '(echo \"%s\"; date) >> %s' % (log_string, JOURNAL_LOG_FILE)
    os.system(command)

    journals = bootstrap.getInfinityPmpDeviceNames()
    for journal in journals:
        command = 'isi_radish -qt %s >> %s' % (journal, JOURNAL_LOG_FILE)
        os.system(command)

def log_drive_info(drives):
    date_done = False
    for info in drives:
        if info == None:
            continue
        if not info.present or not (info.usr_state == drived.USR_ACTIVE or
                                    info.usr_state == drived.USR_INSECURE or
                                    info.usr_state == drived.USR_STALLED):
            continue
        if not date_done:
            date_done = True
            command = 'date >> %s' % RADISH_LOG_FILE
            os.system(command)
        command = 'isi_radish -qt bay%d >> %s' % (info.bay, RADISH_LOG_FILE)
        os.system(command)


# XXX: someday we'll have per-firmware-version settings, so we pass the fwversion
# so that we can look up the right settings and just pass back one dictionary.
def get_drive_config(model, fwversion):
    global drive_configs
    if drive_configs.has_key(model):
        return drive_configs[model]
    else:
        return None

def build_event_cache():
    global celog_event_cache
    celog_event_cache = {}
    bay_event = [
        DRIVEMON_ID,
        SMART_STATUS_ID,
    ]
    model_event = [
        MISSING_CONFIG_ID,
    ]
    event_ids = {
        'bay' : bay_event, 
        'model' : model_event, 
    }
    for event_type, ids in event_ids.iteritems():
        for id in ids:
            celog_event_cache[id] = {}
    events = query_events(event_id=celog_event_cache.keys())
    for event in events:
        event_id = event.event_id
        if event_id in event_ids.get('bay', []):
            specifier = event.specifier.get('bay')
        elif event_id in event_ids.get('model', []):
            specifier = event.specifier.get('model')
        else:
            specifier = None
        add_event_to_cache(event.event_id, specifier, event)

def event_cache_lookup(event_id, specifier):
    return celog_event_cache[event_id].get(specifier, [])

def add_event_to_cache(event_id, specifier, event):
    global celog_event_cache
    if specifier is None:
        return
    events = event_cache_lookup(event_id, specifier) 
    if len(events) == 0:
        celog_event_cache[event_id][specifier] = [event]
    else:
        events = celog_event_cache[event_id][specifier]
        events.append(event)

def cancel_event_in_cache(event_id, specifier, event):
    events = event_cache_lookup(event_id, specifier) 
    if event in events:
        index = events.index(event)
        del events[index]

def update_event_in_cache(event_id, specifier, event):
    events = event_cache_lookup(event_id, specifier) 
    if event in events:
        index = events.index(event)
        events[index] = event

def get_bay_attribute_events(bay, attribute):
    events = []
    bay_events = event_cache_lookup(DRIVEMON_ID, bay)
    for event in bay_events:
        if attribute == event.specifier.get('attribute'):
            events.append(event)
    return events

def generate_event(bay, sev, thresh, monvalue, data,
                   model, firmware, serial, attribute, drive_class, smartfail=False):
    attribute = str(attribute)
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    current_lnn = c.devid_to_lnn(devid)
    specifier = {
        'bay' : bay,
        'attribute' : attribute,
        'model' : model,
        'drive_serial' : serial,
        'firmware' : firmware,
        'devid' : devid,
        'lnn' : current_lnn,
        'threshold' : thresh,
        'device' : drivebay.bay2dev(bay),
    }
    bay_events = get_bay_attribute_events(bay, attribute) 
    # No Events exist for the bay, need to send event 
    if len(bay_events) == 0:
        add_bay_event(specifier, monvalue, sev, thresh, bay, smartfail, drive_class)
    else:
        for event in bay_events:
            # Bug 97351: If an event transitions from warn to critical or
            # is in smartfail cancel and generate a new event so an email
            # notification is generated.
            new_critical = ((event.severity == isi.celog.EVENT_LEVEL_WARN and
                             sev == isi.celog.EVENT_LEVEL_CRIT) or smartfail)
            new_warning = (event.severity == isi.celog.EVENT_LEVEL_CRIT and
                           sev == isi.celog.EVENT_LEVEL_WARN)
            # Bug 99421: Since threshold values can be changed, we need to
            # update/add events if a monitored value has changed or a
            # threshold value has changed.
            if (monvalue != event.value or
                thresh != event.specifier.get('threshold') or new_critical or
                new_warning):
                if not new_critical or new_warning:
                    # Update Event
                    update_event(event, bay, monvalue, sev,
                                 firmware, attribute, thresh, smartfail,
                                 drive_class)
                else:
                    cancel_event(event, bay)
                    add_bay_event(specifier, monvalue, sev, thresh, bay,
                                  smartfail, drive_class)
            else:
                # Do Nothing
                continue

def handle_value(bay, monvalue, data, direction, model, firmware,
                 serial, attribute, handle, drive_class=DriveClass.DATA):
    bay_events = get_bay_attribute_events(bay, attribute)
    # Verify corresponding event has matching model string and
    # serial number for drive bay. If not, new drive is present
    # and old event is cancelled
    for event in bay_events:
        event_specifier = event.specifier
        event_model = event_specifier.get('model')
        event_serial = event_specifier.get('drive_serial')
        if model != event_model and serial != event_serial:
            cancel_event(event, bay)

            # If we sent a journal wear celog alert that means we are in read
            # only. If we have a new serial that means the technician
            # replaced the drive so turn off read only.
            if drive_class == DriveClass.JOURNAL:
                clear_read_only_bit()

    # Check value against thresholds
    # since we normalize the monvalue, we also need to
    # normalize the thresholds
    # A config file DOES NOT have to have all 3 thresholds defined,
    # though note that any left unspecified will receive the gconfig
    # default value of 0.
    if data.has_key('smartfail_threshold'):
        smartthresh = data['smartfail_threshold']
    else:
        smartthresh = None
    if data.has_key('critical_threshold'):
        critthresh = data['critical_threshold']
    else:
        critthresh = None
    if data.has_key('warn_threshold'):
        warnthresh = data['warn_threshold']
    else:
        warnthresh = None
    # Bug 156890 - gconfig will by default always fill in keys that
    # aren't specified.  If all thresholds are the default of 0, then
    # we will treat this attribute as having no thresholds, and skip
    # taking any actions in response to its value.
    x = [data.get('%s_threshold' % key, 0) == 0 for key in ('smartfail','warn','critical')]
    if reduce(lambda a, b: a and b, x): # All are True
        syslog.syslog(syslog.LOG_DEBUG, "Drive %s has default monitor thresholds" % model)
        smartthresh = None
        critthresh = None
        warnthresh = None

    if direction.lower() == "down":
        maximum = data['max']
        if smartthresh != None:
            smartthresh = maximum - smartthresh
        if critthresh != None:
            critthresh = maximum - critthresh
        if warnthresh != None:
            warnthresh = maximum - warnthresh
    if smartthresh != None:
        if monvalue >= smartthresh:
            sev = isi.celog.EVENT_LEVEL_CRIT

            # Don't attempt to smartfail out boot drives or journal drives.
            if drive_class == DriveClass.DATA:
                if handle is not None:
                    send_softfail_msg(handle, "isi_drivemon %s threshold "
                                      "exceeded" % attribute)
                smartfail_msg = True
            elif drive_class == DriveClass.JOURNAL:
                set_read_only_bit()
                smartfail_msg = False
            else: #BOOT
                smartfail_msg = False

            generate_event(bay, sev, smartthresh, monvalue, data,
                           model, firmware, serial, attribute,
                           drive_class, smartfail_msg)
            return
    if critthresh != None:
        if monvalue >= critthresh:
            sev = isi.celog.EVENT_LEVEL_CRIT
            generate_event(bay, sev, critthresh, monvalue, data,
                           model, firmware, serial, attribute, drive_class)

            if drive_class == DriveClass.JOURNAL:
                set_read_only_bit()

            return
    if warnthresh != None:
        if monvalue >= warnthresh:
            sev = isi.celog.EVENT_LEVEL_WARN
            generate_event(bay, sev, warnthresh, monvalue, data,
                           model, firmware, serial, attribute, drive_class)
            return

    # Value specified by drive not above threshold
    if len(bay_events) == 0:
        # No existing event for bay, return
        return
    else:
        # Event exists, we need to cancel these
        for event in bay_events: 
            cancel_event(event, bay)

def handle_smart_status(bay, result, drive_class=DriveClass.DATA):
    bay_events = event_cache_lookup(SMART_STATUS_ID, bay)
    if len(bay_events) == 0:
        # No existing event and haven't exceeded threshold, do nothing
        if not result:
            return
        else:
            add_smart_status_event(bay, drive_class)
    else:
        if not result:
            cancel_smart_status_event(bay)

def handle_event_message(msg, severity, smartfail):
    if severity == isi.celog.EVENT_LEVEL_WARN:
        msg += WARN_ACTION_MSG
    elif severity == isi.celog.EVENT_LEVEL_CRIT and not smartfail:
        msg += CRIT_ACTION_MSG
    elif severity == isi.celog.EVENT_LEVEL_CRIT and smartfail:
        msg += SMARTFAIL_ACTION_MSG
    else:
        msg = msg
    return msg

def cancel_missing_config_event(model):
    config_events = event_cache_lookup(MISSING_CONFIG_ID, model) 
    for event in config_events:
        if model == event.specifier.get('model'):
            cancel_event(event, model)

def cancel_bay_events(bay):
    bay_events = event_cache_lookup(DRIVEMON_ID, bay) 
    for event in bay_events:
        cancel_event(event, bay) 

def cancel_smart_status_event(bay):
    bay_events = event_cache_lookup(SMART_STATUS_ID, bay) 
    for event in bay_events:
        cancel_event(event, bay)

def cancel_event(event, specifier):
    if event is None:
        return
    severity = CELOG_SEVERITIES.get(event.severity)
    message = event.message
    instance_id = event.instance_id
    event_id = event.event_id
    event_displayid = event.display_iid()
    try:
        db = isi.celog.query.Query()
        evt = db.get_events(interactive=True, instanceids=[instance_id])
    except isi.celog.query.QueryError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Cancel_event: error while getting event: %s" % str(e))
        return
    if len(evt) == 0:
        return
    coalesce_instanceid = evt[0].coalesce_instanceid
    if coalesce_instanceid:
        parent = db.get_events(interactive=True, instanceids=[coalesce_instanceid])

    if len(parent) > 0:
        try:
            parent[0].cancel()
            syslog.syslog(syslog.LOG_INFO,
                          "Cancelling Parent Instance ID: %s, Severity: %s, Message: '%s'" % 
                          (coalesce_instanceid, severity, parent[0].message))
            cancel_event_in_cache(event.event_id, specifier, event)
        except SystemError as e:
            syslog.syslog(syslog.LOG_ERR,
                          "Error while cancelling parent event: %s" % str(e))
    else:
        try:
            evt[0].cancel()
            syslog.syslog(syslog.LOG_INFO,
                          "Cancelling Event Instance ID: %s, Severity: %s, Message: '%s'" % 
                          (event_id, severity, evt[0].message))
            cancel_event_in_cache(event.event_id, specifier, event)
        except SystemError as e:
            syslog.syslog(syslog.LOG_ERR,
                          "Error while cancelling event: %s" % str(e))

def update_event(event, bay, value, severity, firmware, attribute, threshold,
                 smartfail, drive_class):
    if event is None:
        return
    try:
        event.value = value 
        event.severity = severity
        event.specifier['firmware'] = firmware
        event.specifier['threshold'] = threshold
        event.message = ("Bay %s %s threshold exceeded: %s (Threshold: %s)." %
                         (bay, attribute, value, threshold))
        if drive_class == DriveClass.BOOT:
            location = drivebay.bay2locnstr(bay)
            if location != "Unknown":
                location = location.strip(' ')
                event.message = ("Drive at %s %s threshold exceeded: %s"
                                 " (Threshold: %s)." %
                                 (location, attribute, value, threshold))
        event.message = handle_event_message(event.message, severity, smartfail)
        severity = CELOG_SEVERITIES.get(event.severity)
        event.update()
        syslog.syslog(syslog.LOG_INFO,
                      "Updating Instance ID: %s, Severity: %s, Message: '%s'" %
                      (event.display_iid(), severity, event.message))
        update_event_in_cache(event.event_id, bay, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def add_smart_status_event(bay, drive_class=DriveClass.DATA):
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    current_lnn = c.devid_to_lnn(devid)
    location = drivebay.bay2locnstr(bay)
    specifier = {
        'lnn' : current_lnn,
        'bay' : bay,
    }
    message = ("Bay %s has exceeded SMART status threshold. This drive" 
               " has been SmartFailed" % bay)
    syslog_bay_str = "bay %s" % bay
    # only fail the data drives (not journal or boot).
    if drive_class != DriveClass.DATA and location != "Unknown":
        location = location.strip(' ')
        message = "Drive at %s has Exceeded SMART status threshold" % location
        syslog_bay_str = "drive at %s" % location
    try:
        event = isi.celog.Event(
            event_id = SMART_STATUS_ID, 
            severity = isi.celog.EVENT_LEVEL_CRIT,
            message = message,
            start_time = int(time.time()),
            specifier = specifier,
            send_event = True,
            devid = devid
        )
        severity = CELOG_SEVERITIES.get(isi.celog.EVENT_LEVEL_CRIT)
        syslog.syslog(syslog.LOG_INFO,
                      "Event ID %s has been sent for %s, Severity: %s,"
                      " Message '%s'" %
                      (event.event_id, syslog_bay_str, severity, message))
        add_event_to_cache(event.event_id, bay, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def add_missing_config_event(model):
    config_events = event_cache_lookup(MISSING_CONFIG_ID, model) 
    for event in config_events:
        if event.specifier == event.specifier.get('model'):
            # Event already exists, do not add
            return
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    current_lnn = c.devid_to_lnn(devid)
    specifier = {
        'lnn' : current_lnn,
        'model' : model,
    }
    try:
        message = "Missing config file for drive model: {model}"
        event = isi.celog.Event(
            event_id = MISSING_CONFIG_ID,
            severity = isi.celog.EVENT_LEVEL_CRIT,
            message = message,
            start_time = int(time.time()),
            specifier = specifier,
            send_event = True,
            devid = devid
        )
        severity = CELOG_SEVERITIES.get(isi.celog.EVENT_LEVEL_CRIT)
        log_msg = ("Missing config file for drive model: %s" %
                   event.specifier.get('model'))
        syslog.syslog(syslog.LOG_INFO,
                      "Event ID %s has been sent for model %s, Severity: %s, Message: '%s'" %
                      (event.event_id, model, severity, log_msg))
        add_event_to_cache(event.event_id, model, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def add_bay_event(specifier, monvalue, sev, thresh, bay, smartfail, drive_class):
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    location = drivebay.bay2locnstr(bay)
    try:
        message = ("Bay %s %s threshold exceeded: %s"
                   " (Threshold: %s)." % (bay, specifier.get('attribute'),
                   monvalue, thresh))
        syslog_bay_str = "bay %s" % bay
        if drive_class == DriveClass.BOOT and location != "Unknown":
            location = location.strip(' ')
            message = ("Drive at %s %s threshold exceeded: %s"
                       " (Threshold: %s)." % (location,
                       specifier.get('attribute'), monvalue, thresh))
            syslog_bay_str = "drive at %s" % location
        message = handle_event_message(message, sev, smartfail)
        event = isi.celog.Event(
            event_id = DRIVEMON_ID,
            severity = sev,
            message = message, 
            start_time = int(time.time()),
            value = monvalue,
            specifier = specifier,
            send_event = True,
            devid = devid,
        )
        severity = CELOG_SEVERITIES.get(sev) 
        syslog.syslog(syslog.LOG_INFO,
                      "Event ID %s has been sent for %s, Severity: %s,"
                      " Message: '%s'" %
                      (event.event_id, syslog_bay_str, severity, message))
        add_event_to_cache(event.event_id, bay, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def query_events(event_id, instance_id=None):
    events = []
    db = isi.celog.query.Query()
    c = isi.app.lib.cluster.Cluster()
    devid = c.local_devid()
    if instance_id is not None:
        instance_ids = [instance_id]
    else:
        instance_ids = None
    if not isinstance(event_id, list):
        event_id = [event_id]
    severities = [
        isi.celog.EVENT_LEVEL_CRIT,
        isi.celog.EVENT_LEVEL_WARN,
    ]
    now = int(time.time())
    DEFAULT_AGE = 30 * 24 * 60 * 60
    oldest = now - DEFAULT_AGE
    newest = now
    attempt = 0
    while attempt < GET_DB_MAX_ATTEMPTS:
        try:
            attempt += 1
            events = db.get_events(severities=severities, acknowledged=False,
                            eventids=event_id, coalesced=None, oldest=oldest,
                            newest=newest, ended=False, instanceids=instance_ids,
                            devices=[devid])
            if isinstance(events, list):
                break
            else:
                time.sleep(GET_DB_WAIT)
        except isi.celog.query.QueryError as e:
            if attempt >= GET_DB_MAX_ATTEMPTS:
                syslog.syslog(syslog.LOG_ERR,
                              "Unable to get database events after %s attempts" %
                               attempt)
            else:
                time.sleep(GET_DB_WAIT)
                continue
    if events is None:
        events = []
    return events

def monitor_drive(model, bay, firmware, serial, handle, drive_class=DriveClass.DATA):
    config = get_drive_config(model, firmware)
    if not config:
        # XXX: when we have all the config files, we want to uncomment this!!!
        # syslog.syslog(syslog.LOG_ERR, "No config file available for drive %s bay %d" % (model, bay))
        # add_missing_config_event(model)
        # time.sleep(ADD_DELAY)
        return
    else:
        cancel_missing_config_event(model)
    disk_type = config['interface_type']
    for d in config['monitor']:
        if d == 'order':
            continue
        attribute = d
        data = config['monitor'][d]
        page = config['monitor'][d]['page']
        offset = config['monitor'][d]['offset']
        length = config['monitor'][d]['length']
        endian = 'big' if config['monitor'][d]['big_endian'] else 'little'
        direction = 'up' if config['monitor'][d]['direction_up'] else 'down'
        syslog.syslog(syslog.LOG_DEBUG,"Model = %s" % model)
        syslog.syslog(syslog.LOG_DEBUG,"offset = %d" % offset)
        syslog.syslog(syslog.LOG_DEBUG,"length = %d" % length)
        syslog.syslog(syslog.LOG_DEBUG,"endian = %s" % endian)
        syslog.syslog(syslog.LOG_DEBUG,"page = %d" % page)
        monval = libdrivemon.getdiskdata(disk_type,endian,length,offset,page,bay,
                                         options.debug and 1 or 0)
        if monval is None:
            syslog.syslog(syslog.LOG_ERR, "unable to read disk data for bay %s"
                          % bay)
        else:
            syslog.syslog(syslog.LOG_DEBUG, "bay %d monval = %d" % (bay, monval))
            if direction.lower() == "up":
                monval = monval
            else:
                monval = config['monitor'][d]['max'] - monval
            handle_value(bay, monval, data, direction, model, firmware,
                         serial, attribute, handle, drive_class)

def get_drives_info(count=WAIT_COUNT, delay=WAIT_SLEEP):
    info = None
    # query isi_drive_d for drives info
    for loop in xrange(count):
        try:
            info = drived.getDrives()
        except:
            # retry waiting for drived to start
            time.sleep(delay)
    if not info:
        syslog.syslog(syslog.LOG_ERR, "DriveD is still not up after %d seconds" % 
                      (delay * count))
    return info

def send_softfail_msg(handle, msg, count=WAIT_COUNT, delay=WAIT_SLEEP):
    # send soft fail (smart fail) msg via drived
    for loop in range(count):
        try:
            drived.driveSoftfailMsg(handle, msg)
            break
        except:
            # retry waiting for drived to start
            time.sleep(delay)
    else:
        syslog.syslog(syslog.LOG_ERR, "DriveD is still not up after %d seconds" % 
                     (delay * count))

def set_read_only_bit():
    cfg = arrcfg.config_load()
    this_node = cfg.find_device_by_devid(cfg.local_devid)
    ro_journal_bit = this_node.read_only & arrcfg.ARR_RO_READ_ONLY_SYSTEM_JOURNAL_WEAR

    if bool(ro_journal_bit):
        return

    cmd = "{} --set {}".format(READ_ONLY_SCRIPT, JOURNAL_RO_BIT)
    err, out = procs.get_cmd_output(cmd)
    if err:
        syslog.syslog(syslog.LOG_ERR,
                      "Drivemon failed to enable node read only mode."
                      "Stderr: {}. Stdout: {}.".format(err, out))
    else:
        syslog.syslog(syslog.LOG_INFO,
                      "Drivemon triggered node read only mode")

def clear_read_only_bit():
    cfg = arrcfg.config_load()
    this_node = cfg.find_device_by_devid(cfg.local_devid)
    ro_journal_bit = this_node.read_only & arrcfg.ARR_RO_READ_ONLY_SYSTEM_JOURNAL_WEAR

    if not bool(ro_journal_bit):
        return

    cmd = "{} --unset {}".format(READ_ONLY_SCRIPT, JOURNAL_RO_BIT)
    err, out = procs.get_cmd_output(cmd)
    if err:
        syslog.syslog(syslog.LOG_ERR,
                      "Drivemon failed to disable node read only mode"
                      "Stderr: {}. Stdout: {}.".format(err, out))
    else:
        syslog.syslog(syslog.LOG_INFO,
                      "Drivemon cleared node read only mode")

def get_boot_drive_model_configs():
    global model_list
    global boot_bay
    global mfginfo_list

    boot_bay = []
    mfginfo_list = []

    if not bootstrap.supportsInternalBootDrives():
        return

    bootdisks = bootstrap.getEligibleBootdiskDeviceNames()
    for bootdisk in bootdisks:
        bay = drivebay.any2bay(bootdisk)
        if bay:
            mfginfo = drivebay.bay2mfginfo(bay)
            if mfginfo == None:
                continue
            mfginfo_list.append(mfginfo)
            model_list.append(mfginfo['model'])
            boot_bay.append(bay)

# Pretty much duplicate code as above. Its more clear this way for now.
def get_journal_drive_model_configs():
    global model_list
    global journal_bays
    global mfginfo_list

    journal_bays = []
    mfginfo_list = []

    drives = bootstrap.getInfinityPmpDeviceNames()
    for drive in drives:
        bay = drivebay.any2bay(drive)
        if bay:
            mfginfo = drivebay.bay2mfginfo(bay)
            if mfginfo is None:
                continue
            mfginfo_list.append(mfginfo)
            model_list.append(mfginfo['model'])
            journal_bays.append(bay)

def load_config_files(drives):
    global drive_configs
    global model_list

    drive_configs = {}
    model_list = []

    get_boot_drive_model_configs()
    if INFINITY:
        get_journal_drive_model_configs()

    # Infinity boot drives will also be in drives list (fyi)
    # but since they are already in model_list they should not
    # be added twice.
    for info in drives:
        if (info.model != '') and (info.model not in model_list):
            model_list.append(info.model)

    all_conf = drive_config.load_drive_configs(model_list)
    for m, c in all_conf.items():
        new_mon = drive_config.valid_drivemon_config(c)
        if new_mon:
            drive_configs[m] = c
            drive_configs[m]['monitor'] = new_mon

def monitor_boot_drives():
    global boot_bay
    global mfginfo_list

    if not bootstrap.supportsInternalBootDrives() and not INFINITY:
        return

    handle = None
    i = 0
    for bay in boot_bay:
        # Query the smart stats for the drive
        unitnum = drivebay.bay2unit(bay)
        mfginfo = mfginfo_list[i]
        result = libdrivemon.monitorSmartStatus(bay, unitnum,
                                                mfginfo['device'])
        handle_smart_status(bay, result, drive_class=DriveClass.BOOT)
        monitor_drive(mfginfo['model'], bay, mfginfo['firmware'],
                      mfginfo['serial'], handle, drive_class=DriveClass.BOOT)
        i = i + 1

def monitor_journal_drives():
    global journal_bays
    global mfginfo_list

    handle = None
    i = 0

    # If len 0 then sysctls couldnt find drive  or we could
    # not find drive info such as config/bay number etc.
    # Go readonly.
    if len(journal_bays) == 0:
        set_read_only_bit()
        return

    for bay in journal_bays:
        # Query the smart stats for the drive
        unitnum = drivebay.bay2unit(bay)
        mfginfo = mfginfo_list[i]
        result = libdrivemon.monitorSmartStatus(bay, unitnum,
                                                mfginfo['device'])
        if result == 1:
            set_read_only_bit()
        else:
            clear_read_only_bit()

        handle_smart_status(bay, result, drive_class=DriveClass.JOURNAL)
        monitor_drive(mfginfo['model'], bay, mfginfo['firmware'],
                      mfginfo['serial'], handle, drive_class=DriveClass.JOURNAL)
        i = i + 1

def monitor_bays(drives):
    bays_monitored = []

    for info in drives:
        if info == None:
            continue
        if not info.present or not (info.usr_state == drived.USR_ACTIVE or 
                                    info.usr_state == drived.USR_INSECURE or
                                    info.usr_state == drived.USR_STALLED):
            continue
        # Query the smart stats for the drive
        result = libdrivemon.monitorSmartStatus(info.bay, info.unit,
                                                info.device)
        if result == 1: 
            send_softfail_msg(info.handle, "SMART status exceeded")
        handle_smart_status(info.bay, result)
        monitor_drive(info.model, info.bay, info.firmware, info.serial, info.handle)
        bays_monitored.append(info.bay)
    maxbays = drivebay.maxBays()
    unchecked_bays = set(range(1, maxbays + 1)) - set(bays_monitored)
    # If the bay is not being checked, cancel event
    for bay in unchecked_bays:
        cancel_bay_events(bay) 
        cancel_smart_status_event(bay)

def monitor_drives():
    global options
    global received_sighup
    global terminate

    monitor_pass = 0
    next_daily_pass_time = 0

    if options.daemonize:
        log_string = "daemon startup"
        pid = daemon.daemonize(pidfilename="/var/run/isi_drivemon.pid", dopid=True,
                               exitfirst=True)
        process.madvise(process.MADV_PROTECT)
        syslog.syslog(syslog.LOG_NOTICE, "Starting daemon")
    else:
        log_string = "one-shot"

    while True:
        # log at startup and log one pass per day.
        # don't log on extra passes triggered by SIGHUP.
        now = time.time()
        if (now >= next_daily_pass_time):
            next_daily_pass_time = now + options.timeout
            log = True
        else:
            log = False

        # get info on every drive bay from isi_drive_d.
        # this won't include boot drives.
        # (On INFINITY this will include boot drives but not journal drives)
        drives = get_drives_info()
        if drives is None:
            # Ignore errors querying drive_d, and hope the next pass
            # has more success.
            drives = []

        received_sighup = False

        if not hal.isVirtualSeries():
            if log:
                syslog.syslog(syslog.LOG_INFO,
                              "monitor_drives: start logging pass %d" % monitor_pass)
                if INFINITY:
                    log_boot_drive_info(log_string)
                    log_journal_drive_info(log_string)
                elif bootstrap.supportsInternalBootDrives():
                    log_boot_drive_info(log_string)
                log_drive_info(drives)
                syslog.syslog(syslog.LOG_INFO,
                              "monitor_drives: finished logging pass %d" % monitor_pass)

            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: start monitor pass %d" % monitor_pass)
            try:
                load_config_files(drives)
                monitor_boot_drives()
                if INFINITY:
                    monitor_journal_drives()
                monitor_bays(drives)
            except:
                unhandled_exception()

            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: finished monitor pass %d" % monitor_pass)
            monitor_pass = monitor_pass + 1

        if not options.daemonize:
            break

        if not received_sighup and not terminate:
            log_string = "daemon periodic"
            sleep_time = next_daily_pass_time - time.time()
            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: sleep_time=%d" % sleep_time)
            time.sleep(sleep_time)
            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: woke with sighup=%s terminate=%s" %
                          (received_sighup, terminate))
        else:
            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: not sleeping due to sighup=%s terminate=%s" %
                          (received_sighup, terminate))

        if terminate:
            syslog.syslog(syslog.LOG_INFO, "termination requested")
            # log boot drive info again before terminating, to help
            # with diagnosing boot drive problems.  We want numbers
            # recorded from sysctls, since we can't trust the SMART
            # stats on the Sandisk drives.
            if INFINITY:
                log_boot_drive_info("daemon termination")
                log_journal_drive_info("daemon termination")
            elif bootstrap.supportsInternalBootDrives():
                log_boot_drive_info("daemon termination")
            break

    if options.daemonize:
        syslog.syslog(syslog.LOG_NOTICE, "Exiting daemon")

def main():
    global options
    global received_sighup
    global terminate
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose',  action="store_true", dest="debug",
                      default=False, help='enable debug output')
    parser.add_option('-d', '--daemonize',  action="store_true", dest="daemonize",
                      default=False, help='run as daemon with default time out 24 hours')
    parser.add_option('-t', '--timeout', action='store', dest='timeout', 
                      default=DEFAULT_SLEEP, type='int', 
                      help='run as daemon with specified timeout')
    (options, args) = parser.parse_args()

    terminate = False
    received_sighup = False

    libdrivemon.initDrivemonLib()

    syslog.openlog("isi_drivemon", syslog.LOG_PID, syslog.LOG_DAEMON)
    if options.debug:
        syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG))
    else:
        syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_INFO))

    signal.signal(signal.SIGINT, signal_handler_sigint)
    signal.signal(signal.SIGTERM, signal_handler_sigterm)
    signal.signal(signal.SIGHUP, signal_handler_sighup)

    build_event_cache()

    monitor_drives()

    libdrivemon.cleanupDrivemonLib()
    syslog.closelog()


if __name__ == '__main__':
    main()


