#!/usr/libexec/isilon/isi_drivemon

import isi.app.lib.cluster
import isi.app.daemon as daemon
import isi.app.lib.process as process
import isi.celog.eventids as eventids
import isi.celog.query
import isi.hw.bay as drivebay
import isi.hw.drived as drived
import isi.hw.drivemon as libdrivemon
import isi.hw.hal as hal
import isi.hw.drive_config as drive_config
import inspect
import optparse
import os 
import signal
import sys
import syslog
import time
import traceback

DRIVEMON_ID = eventids.eventid_map["SYS_DRIVEMON"]
MISSING_CONFIG_ID = eventids.eventid_map["SYS_DRIVEMON_MISSING_CONFIG"]
SMART_STATUS_ID = eventids.eventid_map["SYS_DRIVEMON_SMARTSTATUS"]
ADD_DELAY = 0.2


RADISH_LOG_FILE	= "/var/log/isi_radish.log"
BOOTDISK_LOG_FILE = "/var/log/isi_radish_bootdisk.log"
ATA_LOG_FILE  = "/var/log/hw_ata.log"

# 86400 seconds
DEFAULT_SLEEP = (24 * 60 * 60)

WAIT_TIME = 60
WAIT_SLEEP = 10
WAIT_COUNT = (WAIT_TIME / WAIT_SLEEP)

GET_DB_WAIT = 5
GET_DB_MAX_ATTEMPTS = 2

MAX_CANCEL_ATTEMPTS = 10
CANCEL_DELAY = 0.1

drive_configs = {}
model_list = []

mfginfo_list = []
boot_bay = []

sighup_count = 1
terminate = False

celog_event_cache = {}

CELOG_SEVERITIES = {
    isi.celog.EVENT_LEVEL_CRIT : 'Critical',
    isi.celog.EVENT_LEVEL_WARN : 'Warning',
    }

WARN_ACTION_MSG = " Please schedule drive replacement."
CRIT_ACTION_MSG = " Please schedule drive replacement immediately."
SMARTFAIL_ACTION_MSG = (" Drive is being removed from the system -"
                        " please schedule drive replacement immediately.")

CallerLine = lambda: inspect.stack()[2][2] 
CallerName = lambda: inspect.stack()[2][3]

def unhandled_exception():
    etype, evalue, etb = sys.exc_info()
    syslog.syslog(syslog.LOG_ERR,
                    "Exception caught during %s (line %s): %s, %s\n" %
		    (CallerName(), CallerLine(), etype, evalue))
    # syslog.syslog(syslog.LOG_INFO,
    #                ("Caller: %s:%s\n" % (CallerName(), CallerLine())))
    lines = traceback.format_exception(etype, evalue, etb)
    lines = reduce(lambda a,b: a + b, map(lambda s: s.splitlines(), lines))
    for line in lines:
        syslog.syslog(syslog.LOG_INFO, "%s\n" % line)


def signal_handler_sigterm(signum, frame):
    global terminate
    syslog.syslog(syslog.LOG_INFO, "Received SIGTERM")
    terminate = True

def signal_handler_sigint(signum, frame):
    global terminate
    syslog.syslog(syslog.LOG_INFO, "Received SIGINT")
    terminate = True

def signal_handler_sighup(signum, frame):
    global received_sighup
    global sighup_count
    received_sighup = True
    syslog.syslog(syslog.LOG_INFO, "Received SIGHUP #%d" % sighup_count)
    sighup_count = sighup_count + 1

def log_boot_drive_info(log_string):
    command = '(echo \"%s\"; date) >> %s' % (log_string, ATA_LOG_FILE)
    os.system(command)

    command = 'sysctl hw.ata | grep -v "use -D" | grep -v inject_ >> %s' % ATA_LOG_FILE
    os.system(command)

    command = '(echo \"%s\"; date) >> %s' % (log_string, BOOTDISK_LOG_FILE)
    os.system(command)

    bootdisks = hal.getBootdiskDeviceNames()
    for bootdisk in bootdisks:
        command = 'isi_radish -qt %s >> %s' % (bootdisk, BOOTDISK_LOG_FILE)
        os.system(command)


def log_drive_info(drives):
    date_done = False
    for info in drives:
        if info == None:
            continue
        if not info.present or not (info.usr_state == drived.USR_ACTIVE or
                                    info.usr_state == drived.USR_INSECURE or
                                    info.usr_state == drived.USR_STALLED):
            continue
        if not date_done:
            date_done = True
            command = 'date >> %s' % RADISH_LOG_FILE
            os.system(command)
        command = 'isi_radish -qt bay%d >> %s' % (info.bay, RADISH_LOG_FILE)
        os.system(command)


# XXX: someday we'll have per-firmware-version settings, so we pass the fwversion
# so that we can look up the right settings and just pass back one dictionary.
def get_drive_config(model, fwversion):
    global drive_configs
    if drive_configs.has_key(model):
        return drive_configs[model]
    else:
        return None

def build_event_cache():
    global celog_event_cache
    celog_event_cache = {}
    bay_event = [
        DRIVEMON_ID,
        SMART_STATUS_ID,
    ]
    model_event = [
        MISSING_CONFIG_ID,
    ]
    event_ids = {
        'bay' : bay_event, 
        'model' : model_event, 
    }
    for event_type, ids in event_ids.iteritems():
        for id in ids:
            celog_event_cache[id] = {}
    events = query_events(event_id=celog_event_cache.keys())
    for event in events:
        event_id = event.event_id
        if event_id in event_ids.get('bay', []):
            specifier = event.specifier.get('bay')
        elif event_id in event_ids.get('model', []):
            specifier = event.specifier.get('model')
        else:
            specifier = None
        add_event_to_cache(event.event_id, specifier, event)

def event_cache_lookup(event_id, specifier):
    return celog_event_cache[event_id].get(specifier, [])

def add_event_to_cache(event_id, specifier, event):
    global celog_event_cache
    if specifier is None:
        return
    events = event_cache_lookup(event_id, specifier) 
    if len(events) == 0:
        celog_event_cache[event_id][specifier] = [event]
    else:
        events = celog_event_cache[event_id][specifier]
        events.append(event)

def cancel_event_in_cache(event_id, specifier, event):
    events = event_cache_lookup(event_id, specifier) 
    if event in events:
        index = events.index(event)
        del events[index]

def update_event_in_cache(event_id, specifier, event):
    events = event_cache_lookup(event_id, specifier) 
    if event in events:
        index = events.index(event)
        events[index] = event

def get_bay_attribute_events(bay, attribute):
    events = []
    bay_events = event_cache_lookup(DRIVEMON_ID, bay)
    for event in bay_events:
        if attribute == event.specifier.get('attribute'):
            events.append(event)
    return events

def generate_event(bay, sev, thresh, monvalue, data, 
                   model, firmware, serial, attribute, boot_drive, smartfail=False):
    attribute = str(attribute)
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    current_lnn = c.devid_to_lnn(devid)
    specifier = {
        'bay' : bay,
        'attribute' : attribute,
        'model' : model,
        'drive_serial' : serial,
        'firmware' : firmware,
        'devid' : devid,
        'lnn' : current_lnn,
        'threshold' : thresh,
    }
    bay_events = get_bay_attribute_events(bay, attribute) 
    # No Events exist for the bay, need to send event 
    if len(bay_events) == 0:
        add_bay_event(specifier, monvalue, sev, thresh, bay, smartfail, boot_drive)
    else:
        for event in bay_events:
            # Bug 97351: If an event transitions from warn to critical or
            # is in smartfail cancel and generate a new event so an email
            # notification is generated.
            new_critical = ((event.severity == isi.celog.EVENT_LEVEL_WARN and
                             sev == isi.celog.EVENT_LEVEL_CRIT) or smartfail)
            new_warning = (event.severity == isi.celog.EVENT_LEVEL_CRIT and
                           sev == isi.celog.EVENT_LEVEL_WARN)
            # Bug 99421: Since threshold values can be changed, we need to
            # update/add events if a monitored value has changed or a
            # threshold value has changed.
            if (monvalue != event.value or
                thresh != event.specifier.get('threshold') or new_critical or
                new_warning):
                if not new_critical or new_warning:
                    # Update Event
                    update_event(event, bay, monvalue, sev,
                                 firmware, attribute, thresh, smartfail,
                                 boot_drive)
                else:
                    cancel_event(event, bay)
                    add_bay_event(specifier, monvalue, sev, thresh, bay,
                                  smartfail, boot_drive)
            else:
                # Do Nothing
                continue

def handle_value(bay, monvalue, data, direction, model, firmware, 
                 serial, attribute, handle, boot_drive=False):
    bay_events = get_bay_attribute_events(bay, attribute)
    # Verify corresponding event has matching model string and
    # serial number for drive bay. If not, new drive is present
    # and old event is cancelled
    for event in bay_events:
        event_specifier = event.specifier
        event_model = event_specifier.get('model')
        event_serial = event_specifier.get('drive_serial')
        if model != event_model and serial != event_serial:
            cancel_event(event, bay)
    # Check value against thresholds
    # since we normalize the monvalue, we also need to
    # normalize the thresholds
    # A config file DOES NOT have to have all 3 thresholds defined.
    # If we want to disable a threshold, remove it from the config file.
    if data.has_key('smartfail-threshold'):
        smartthresh = data['smartfail-threshold']
    else:
        smartthresh = None
    if data.has_key('critical-threshold'):
        critthresh = data['critical-threshold']
    else:
        critthresh = None
    if data.has_key('warn-threshold'):
        warnthresh = data['warn-threshold']
    else:
        warnthresh = None
    if direction.lower() == "down":
        maximum = data['max']
        if smartthresh != None:
            smartthresh = maximum - smartthresh
        if critthresh != None:
            critthresh = maximum - critthresh
        if warnthresh != None:
            warnthresh = maximum - warnthresh
    if smartthresh != None:
        if monvalue >= smartthresh:
            sev = isi.celog.EVENT_LEVEL_CRIT
            if handle is not None:
                send_softfail_msg(handle, "isi_drivemon %s threshold exceeded" % attribute) 
            generate_event(bay, sev, smartthresh, monvalue, data,
                           model, firmware, serial, attribute, boot_drive, smartfail=True)
            return
    if critthresh != None:
        if monvalue >= critthresh:
            sev = isi.celog.EVENT_LEVEL_CRIT
            generate_event(bay, sev, critthresh, monvalue, data,
                           model, firmware, serial, attribute, boot_drive)
            return
    if warnthresh != None:
        if monvalue >= warnthresh:
            sev = isi.celog.EVENT_LEVEL_WARN
            generate_event(bay, sev, warnthresh, monvalue, data, 
                           model, firmware, serial, attribute, boot_drive)
            return

    # Value specified by drive not above threshold
    if len(bay_events) == 0:
        # No existing event for bay, return
        return
    else:
        # Event exists, we need to cancel these
        for event in bay_events: 
            cancel_event(event, bay)

def handle_smart_status(bay, result, boot_drive=False):
    bay_events = event_cache_lookup(SMART_STATUS_ID, bay) 
    if len(bay_events) == 0: 
        # No existing event and haven't exceeded threshold, do nothing
        if not result:
            return
        else:
            add_smart_status_event(bay, boot_drive) 
    else:
        if not result:
            cancel_smart_status_event(bay)

def handle_event_message(msg, severity, smartfail):
    if severity == isi.celog.EVENT_LEVEL_WARN:
        msg += WARN_ACTION_MSG
    elif severity == isi.celog.EVENT_LEVEL_CRIT and not smartfail:
        msg += CRIT_ACTION_MSG
    elif severity == isi.celog.EVENT_LEVEL_CRIT and smartfail:
        msg += SMARTFAIL_ACTION_MSG
    else:
        msg = msg
    return msg

def cancel_missing_config_event(model):
    config_events = event_cache_lookup(MISSING_CONFIG_ID, model) 
    for event in config_events:
        if model == event.specifier.get('model'):
            cancel_event(event, model)

def cancel_bay_events(bay):
    bay_events = event_cache_lookup(DRIVEMON_ID, bay) 
    for event in bay_events:
        cancel_event(event, bay) 

def cancel_smart_status_event(bay):
    bay_events = event_cache_lookup(SMART_STATUS_ID, bay) 
    for event in bay_events:
        cancel_event(event, bay)

def cancel_event(event, specifier):
    if event is None:
        return
    severity = CELOG_SEVERITIES.get(event.severity)
    message = event.message
    instance_id = event.instance_id
    event_id = event.event_id
    event_displayid = event.display_iid()
    try:
        db = isi.celog.query.Query()
        evt = db.get_events(interactive=True, instanceids=[instance_id])
    except isi.celog.query.QueryError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Cancel_event: error while getting event: %s" % str(e))
        return
    if len(evt) == 0:
        return
    coalesce_instanceid = evt[0].coalesce_instanceid
    if coalesce_instanceid:
        parent = db.get_events(interactive=True, instanceids=[coalesce_instanceid])
        try:
            parent[0].cancel()
            syslog.syslog(syslog.LOG_INFO,
                          "Cancelling Parent Instance ID: %s, Severity: %s, Message: '%s'" % 
                          (coalesce_instanceid, severity, parent[0].message))
            cancel_event_in_cache(event.event_id, specifier, event)
        except SystemError as e:
            syslog.syslog(syslog.LOG_ERR,
                          "Error while cancelling parent event: %s" % str(e))
    else:
        try:
            evt[0].cancel()
            syslog.syslog(syslog.LOG_INFO,
                          "Cancelling Event Instance ID: %s, Severity: %s, Message: '%s'" % 
                          (event_id, severity, evt[0].message))
            cancel_event_in_cache(event.event_id, specifier, event)
        except SystemError as e:
            syslog.syslog(syslog.LOG_ERR,
                          "Error while cancelling event: %s" % str(e))

def update_event(event, bay, value, severity, firmware, attribute, threshold,
                 smartfail, boot_drive):
    if event is None:
        return
    try:
        event.value = value 
        event.severity = severity
        event.specifier['firmware'] = firmware
        event.specifier['threshold'] = threshold
        event.message = ("Bay %s %s threshold exceeded: %s (Threshold: %s)." %
                        (bay, attribute, value, threshold))
        if boot_drive:
            location = drivebay.bay2locnstr(bay)
            if location != "Unknown":
                location = location.strip(' ')
                event.message = ("Drive at %s %s threshold exceeded: %s"
                                 " (Threshold: %s)." %
                                 (location, attribute, value, threshold))
        event.message = handle_event_message(event.message, severity, smartfail)
        severity = CELOG_SEVERITIES.get(event.severity)
        event.update()
        syslog.syslog(syslog.LOG_INFO,
                      "Updating Instance ID: %s, Severity: %s, Message: '%s'" %
                      (event.display_iid(), severity, event.message))
        update_event_in_cache(event.event_id, bay, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def add_smart_status_event(bay, boot_drive=False):
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    current_lnn = c.devid_to_lnn(devid)
    location = drivebay.bay2locnstr(bay)
    specifier = {
        'lnn' : current_lnn,
        'bay' : bay,
    }
    message = ("Bay %s has exceeded SMART status threshold. This drive" 
               " has been SmartFailed" % bay)
    syslog_bay_str = "bay %s" % bay
    if boot_drive and location != "Unknown":
        location = location.strip(' ')
        message = "Drive at %s has Exceeded SMART status threshold" % location
        syslog_bay_str = "drive at %s" % location
    try:
        event = isi.celog.Event(
            event_id = SMART_STATUS_ID, 
            severity = isi.celog.EVENT_LEVEL_CRIT,
            message = message,
            start_time = int(time.time()),
            specifier = specifier,
            send_event = True,
            devid = devid
        )
        severity = CELOG_SEVERITIES.get(isi.celog.EVENT_LEVEL_CRIT)
        syslog.syslog(syslog.LOG_INFO,
                      "Event ID %s has been sent for %s, Severity: %s,"
                      " Message '%s'" %
                      (event.event_id, syslog_bay_str, severity, message))
        add_event_to_cache(event.event_id, bay, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def add_missing_config_event(model):
    config_events = event_cache_lookup(MISSING_CONFIG_ID, model) 
    for event in config_events:
        if event.specifier == event.specifier.get('model'):
            # Event already exists, do not add
            return
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    current_lnn = c.devid_to_lnn(devid)
    specifier = {
        'lnn' : current_lnn,
        'model' : model,
    }
    try:
        message = "Missing config file for drive model: {model}"
        event = isi.celog.Event(
            event_id = MISSING_CONFIG_ID,
            severity = isi.celog.EVENT_LEVEL_CRIT,
            message = message,
            start_time = int(time.time()),
            specifier = specifier,
            send_event = True,
            devid = devid
        )
        severity = CELOG_SEVERITIES.get(isi.celog.EVENT_LEVEL_CRIT)
        log_msg = ("Missing config file for drive model: %s" %
                   event.specifier.get('model'))
        syslog.syslog(syslog.LOG_INFO,
                      "Event ID %s has been sent for model %s, Severity: %s, Message: '%s'" %
                      (event.event_id, model, severity, log_msg))
        add_event_to_cache(event.event_id, model, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def add_bay_event(specifier, monvalue, sev, thresh, bay, smartfail, boot_drive):
    c = isi.app.lib.cluster.Cluster()
    devid = int(c.local_devid())
    location = drivebay.bay2locnstr(bay)
    try:
        message = ("Bay %s %s threshold exceeded: %s"
                   " (Threshold: %s)." % (bay, specifier.get('attribute'),
                   monvalue, thresh))
        syslog_bay_str = "bay %s" % bay
        if boot_drive and location != "Unknown":
            location = location.strip(' ')
            message = ("Drive at %s %s threshold exceeded: %s"
                       " (Threshold: %s)." % (location,
                       specifier.get('attribute'), monvalue, thresh))
            syslog_bay_str = "drive at %s" % location
        message = handle_event_message(message, sev, smartfail)
        event = isi.celog.Event(
            event_id = DRIVEMON_ID,
            severity = sev,
            message = message, 
            start_time = int(time.time()),
            value = monvalue,
            specifier = specifier,
            send_event = True,
            devid = devid,
        )
        severity = CELOG_SEVERITIES.get(sev) 
        syslog.syslog(syslog.LOG_INFO,
                      "Event ID %s has been sent for %s, Severity: %s,"
                      " Message: '%s'" %
                      (event.event_id, syslog_bay_str, severity, message))
        add_event_to_cache(event.event_id, bay, event)
    except SystemError as e:
        syslog.syslog(syslog.LOG_ERR,
                      "Error while sending event: %s" % str(e))

def query_events(event_id, instance_id=None):
    events = []
    db = isi.celog.query.Query()
    c = isi.app.lib.cluster.Cluster()
    devid = c.local_devid()
    if instance_id is not None:
        instance_ids = [instance_id]
    else:
        instance_ids = None
    if not isinstance(event_id, list):
        event_id = [event_id]
    severities = [
        isi.celog.EVENT_LEVEL_CRIT,
        isi.celog.EVENT_LEVEL_WARN,
    ]
    now = int(time.time())
    DEFAULT_AGE = 30 * 24 * 60 * 60
    oldest = now - DEFAULT_AGE
    newest = now
    attempt = 0
    while attempt < GET_DB_MAX_ATTEMPTS:
        try:
            attempt += 1
            events = db.get_events(severities=severities, acknowledged=False,
                            eventids=event_id, coalesced=None, oldest=oldest,
                            newest=newest, ended=False, instanceids=instance_ids,
                            devices=[devid])
            if isinstance(events, list):
                break
            else:
                time.sleep(GET_DB_WAIT)
        except isi.celog.query.QueryError as e:
            if attempt >= GET_DB_MAX_ATTEMPTS:
                syslog.syslog(syslog.LOG_ERR,
                              "Unable to get database events after %s attempts" %
                               attempt)
            else:
                time.sleep(GET_DB_WAIT)
                continue
    if events is None:
        events = []
    return events

def monitor_drive(model, bay, firmware, serial, handle, boot_drive=False):
    config = get_drive_config(model, firmware)
    if not config:
        # XXX: when we have all the config files, we want to uncomment this!!!
        # syslog.syslog(syslog.LOG_ERR, "No config file available for drive %s bay %d" % (model, bay))
        # add_missing_config_event(model)
        # time.sleep(ADD_DELAY)
        return
    else:
        cancel_missing_config_event(model)
    disk_type = config['type']
    for d in config['monitor']:
        if d == 'order':
            continue
        attribute = d
        data = config['monitor'][d]
        page = config['monitor'][d]['page']
        offset = config['monitor'][d]['offset']
        length = config['monitor'][d]['length']
        endian = config['monitor'][d]['endian']
        direction = config['monitor'][d]['direction']
        syslog.syslog(syslog.LOG_DEBUG,"Model = %s" % model)
        syslog.syslog(syslog.LOG_DEBUG,"offset = %d" % offset)
        syslog.syslog(syslog.LOG_DEBUG,"length = %d" % length)
        syslog.syslog(syslog.LOG_DEBUG,"endian = %s" % endian)
        syslog.syslog(syslog.LOG_DEBUG,"page = %d" % page)
        monval = libdrivemon.getdiskdata(disk_type,endian,length,offset,page,bay,
                                         options.debug and 1 or 0)
        if monval is None:
            syslog.syslog(syslog.LOG_ERR, "unable to read disk data for bay %s"
                          % bay)
        else:
            syslog.syslog(syslog.LOG_DEBUG,"monval = %d" % monval)
            if direction.lower() == "up":
                monval = monval
            else:
                monval = config['monitor'][d]['max'] - monval
            handle_value(bay, monval, data, direction, model, firmware, 
                         serial, attribute, handle, boot_drive)

def get_drives_info(count=WAIT_COUNT, delay=WAIT_SLEEP):
    info = None
    # query isi_drive_d for drives info
    for loop in xrange(count):
        try:
            info = drived.getDrives()
        except:
            # retry waiting for drived to start
            time.sleep(delay)
    if not info:
        syslog.syslog(syslog.LOG_ERR, "DriveD is still not up after %d seconds" % 
                      (delay * count))
    return info

def send_softfail_msg(handle, msg, count=WAIT_COUNT, delay=WAIT_SLEEP):
    # send soft fail (smart fail) msg via drived
    for loop in range(count):
        try:
            drived.driveSoftfailMsg(handle, msg)
            break
        except:
            # retry waiting for drived to start
            time.sleep(delay)
    else:
        syslog.syslog(syslog.LOG_ERR, "DriveD is still not up after %d seconds" % 
                     (delay * count))

def get_boot_drive_model_configs():
    global model_list
    global boot_bay
    global mfginfo_list

    boot_bay = []
    mfginfo_list = []

    if not hal.supportsInternalBootDrives():
        return

    bootdisks = hal.getBootdiskDeviceNames()
    for bootdisk in bootdisks:
        bay = drivebay.any2bay(bootdisk)
        if bay:
            mfginfo = drivebay.bay2mfginfo(bay)
            if mfginfo == None:
                continue
            mfginfo_list.append(mfginfo)
            model_list.append(mfginfo['model'])
            boot_bay.append(bay)

def load_config_files(drives):
    global drive_configs
    global model_list

    model_list = []

    get_boot_drive_model_configs()

    for info in drives:
        if (info.model != '') and (info.model not in model_list):
            model_list.append(info.model)

    drive_configs = drive_config.load_drive_configs(model_list)
    for config in drive_configs:
        if not drive_config.valid_drivemon_config(drive_configs[config]):
            drive_configs.pop(config)

def monitor_boot_drives():
    global boot_bay
    global mfginfo_list

    if not hal.supportsInternalBootDrives():
        return

    handle = None
    i = 0
    for bay in boot_bay:
        # Query the smart stats for the drive
        unitnum = drivebay.bay2unit(bay)
        mfginfo = mfginfo_list[i]
        result = libdrivemon.monitorSmartStatus(bay, unitnum,
                                                mfginfo['device'])
        handle_smart_status(bay, result, boot_drive=True)
        monitor_drive(mfginfo['model'], bay, mfginfo['firmware'],
                      mfginfo['serial'], handle, boot_drive=True)
        i = i + 1

def monitor_bays(drives):
    bays_monitored = []

    for info in drives:
        if info == None:
            continue
        if not info.present or not (info.usr_state == drived.USR_ACTIVE or 
                                    info.usr_state == drived.USR_INSECURE or
                                    info.usr_state == drived.USR_STALLED):
            continue
        # Query the smart stats for the drive
        result = libdrivemon.monitorSmartStatus(info.bay, info.unit,
                                                info.device)
        if result == 1: 
            send_softfail_msg(info.handle, "SMART status exceeded")
        handle_smart_status(info.bay, result)
        monitor_drive(info.model, info.bay, info.firmware, info.serial, info.handle)
        bays_monitored.append(info.bay)
    maxbays = drivebay.maxBays()
    unchecked_bays = set(range(1, maxbays + 1)) - set(bays_monitored)
    # If the bay is not being checked, cancel event
    for bay in unchecked_bays:
        cancel_bay_events(bay) 
        cancel_smart_status_event(bay)

def monitor_drives():
    global options
    global received_sighup
    global terminate

    monitor_pass = 0
    next_daily_pass_time = 0

    if options.daemonize:
        log_string = "daemon startup"
        pid = daemon.daemonize(pidfilename="/var/run/isi_drivemon.pid", dopid=True,
                               exitfirst=True)
        process.madvise(process.MADV_PROTECT)
        syslog.syslog(syslog.LOG_NOTICE, "Starting daemon")
    else:
        log_string = "one-shot"

    while True:
        # log at startup and log one pass per day.
        # don't log on extra passes triggered by SIGHUP.
        now = time.time()
        if (now >= next_daily_pass_time):
            next_daily_pass_time = now + options.timeout
            log = True
        else:
            log = False

        # get info on every drive bay from isi_drive_d.
        # this won't include boot drives.
        drives = get_drives_info()
        if drives is None:
            # Ignore errors querying drive_d, and hope the next pass
            # has more success.
            drives = []

        received_sighup = False

        if not hal.isVirtualSeries():
            if log:
                syslog.syslog(syslog.LOG_INFO,
                              "monitor_drives: start logging pass %d" % monitor_pass)
                if hal.supportsInternalBootDrives():
                    log_boot_drive_info(log_string)
                log_drive_info(drives)
                syslog.syslog(syslog.LOG_INFO,
                              "monitor_drives: finished logging pass %d" % monitor_pass)

            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: start monitor pass %d" % monitor_pass)
            try:
                load_config_files(drives)
                monitor_boot_drives()
                monitor_bays(drives)
            except:
                unhandled_exception()

            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: finished monitor pass %d" % monitor_pass)
            monitor_pass = monitor_pass + 1

        if not options.daemonize:
            break

        if not received_sighup and not terminate:
            log_string = "daemon periodic"
            sleep_time = next_daily_pass_time - time.time()
            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: sleep_time=%d" % sleep_time)
            time.sleep(sleep_time)
            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: woke with sighup=%s terminate=%s" %
                          (received_sighup, terminate))
        else:
            syslog.syslog(syslog.LOG_INFO,
                          "monitor_drives: not sleeping due to sighup=%s terminate=%s" %
                          (received_sighup, terminate))

        if terminate:
            syslog.syslog(syslog.LOG_INFO, "termination requested")
            # log boot drive info again before terminating, to help
            # with diagnosing boot drive problems.  We want numbers
            # recorded from sysctls, since we can't trust the SMART
            # stats on the Sandisk drives.
            if hal.supportsInternalBootDrives():
                log_boot_drive_info("daemon termination")
            break

    if options.daemonize:
        syslog.syslog(syslog.LOG_NOTICE, "Exiting daemon")

def main():
    global options
    global received_sighup
    global terminate
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose',  action="store_true", dest="debug",
                      default=False, help='enable debug output')
    parser.add_option('-d', '--daemonize',  action="store_true", dest="daemonize",
                      default=False, help='run as daemon with default time out 24 hours')
    parser.add_option('-t', '--timeout', action='store', dest='timeout', 
                      default=DEFAULT_SLEEP, type='int', 
                      help='run as daemon with specified timeout')
    (options, args) = parser.parse_args()

    terminate = False
    received_sighup = False

    libdrivemon.initDrivemonLib()

    syslog.openlog("isi_drivemon", syslog.LOG_PID, syslog.LOG_DAEMON)
    if options.debug:
        syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG))
    else:
        syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_INFO))

    signal.signal(signal.SIGINT, signal_handler_sigint)
    signal.signal(signal.SIGTERM, signal_handler_sigterm)
    signal.signal(signal.SIGHUP, signal_handler_sighup)

    build_event_cache()

    monitor_drives()

    libdrivemon.cleanupDrivemonLib()
    syslog.closelog()


if __name__ == '__main__':
    main()


