#!/usr/bin/env python

'''
Migrate /var to data drives on Gen4/Gen5 platforms.

Also, handle rebalances, reverse migrations, and enable or disable
excessive boot flash writing alert based on var migration status.
'''

import fcntl
import optparse
import os
import sys
import syslog

import isi.app.lib.cluster as cluster
import isi.app.lib.procs as procs
import isi.celog.celog_config_api as ccapi
import isi.celog.eventids as eventids
import isi.celog.query as query
import isi.config.ifs as ifs_config
import isi.misc as misc
import isi.hw.bay as dbay
import isi.hw.bootstrap as bootstrap
import isi.hw.drived as drived
import isi.sys.bootdisk as bootdisk
import isi.sys.geom as geom

VAR_LKGS = [ '%s%s' % (bootdisk.PROVIDER_ID_VAR, n)
             for n in bootdisk.lkg_suffixes ]
LOCAL_GC_CMD = ('/usr/bin/isi_gconfig -t celog_local '
                'settings.boot_flash_write_rate.%s')

def log_print_exit(msg_lvl=syslog.LOG_NOTICE, msg=None, exit_val=None):
    ''' Logging/printing/exiting helper function '''
    if msg:
        syslog.syslog(msg_lvl, msg.strip('\n'))
        print msg.strip('\n')
    if type(exit_val) == int:
        sys.exit(exit_val)

def option_parser_setup():
    ''' Set up option parser to be used in main '''
    parser = optparse.OptionParser()
    parser.add_option('--status', action='store_true',
                      help='display var mirror status and settings')
    parser.add_option('--migrate', action='store_true',
                      help='migrate var to data drives')
    parser.add_option('--rebalance', action='store_true',
                      help='create additional copies if necessary')
    parser.add_option('--undo', action='store_true',
                      help='migrate var back to boot flash drives')
    parser.add_option('--copies', type='int', metavar='N',
                      help='number of copies to maintain')
    parser.add_option('--include', metavar='[drives]',
                      help='comma separated list of drives to '
                      'prefer to use in migrate or rebalance action')
    parser.add_option('--exclude', metavar='[drives]',
                      help='comma separated list of drives not '
                      'to use in migrate or rebalance action')
    parser.add_option('--enable', action='store_true',
                      help='globally enable var migration')
    parser.add_option('--disable', action='store_true',
                      help='globally disable var migration')
    parser.add_option('--threshold', type='int', metavar='N',
                      help='GB/Day threshold for excessive writing event')
    parser.add_option('--duration', type='int', metavar='N',
                      help='Hour threshold for excessive writing event')
    return parser

def status(settings):
    print '\nMigration Status'
    reqs = [ r for r in bootdisk.bootdisk_reqs if r.label in VAR_LKGS ]
    for req in reqs:
        try:
            mg = bootdisk.mesh.mirror_geom(req.label)
        except KeyError:
            log_print_exit(msg_lvl=syslog.LOG_ERR,
                           msg='Failed to find %s mirror geom' % req.label)
            continue
        storage_geoms = [ mc.provider.geom for mc in mg.consumers
                          if mc.provider.name.startswith('da') ]
        bootflash_geoms = [ mc.provider.geom for mc in mg.consumers
                            if mc.provider.name.startswith('ad') ]
        migration_status = 'ERROR'
        if len(storage_geoms) > 0 and len(bootflash_geoms) == 0:
            migration_status = 'migrated'
        elif len(storage_geoms) == 0 and len(bootflash_geoms) > 0:
            migration_status = 'not migrated'
        elif len(storage_geoms) > 0 and len(bootflash_geoms) > 0:
            migration_status = 'partially migrated'
        print '    %s: %s' % (mg.name, migration_status)
    print '\nMigration Settings'
    print '    enabled: %s' % settings.get('enabled')
    print '     copies: %d' % settings.get('var_copies')
    print '\nExcessive Writing Event Settings'
    print '       write threshold: %d GB/Day' % settings.get('threshold')
    print '    duration threshold: %d hours\n' % settings.get('duration')

def get_geoms(opts):
    ''' Return a list of storage drive partition geoms valid for migration '''
    usr_states = {}
    assume_usr_active = False
    try:
        lnn = cluster.Device().lnn()
        for drive in drived.getDrives(lnn):
            if drive.present:
                usr_states[drive.bay] = drive.usr_state
    except (drived.Error, ifs_config.error):
        log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Failed to query drive_d')
        # Assume all drives are USR_ACTIVE if we can't contact drive_d
        assume_usr_active = True
    eligible_geoms = []
    for geom in bootdisk.mesh.partition_geoms:
        var_count = len([ p for p in geom.providers if p.label in VAR_LKGS ])
        bay = dbay.dev2bay(geom.name)
        (lnum, active, softfail) = dbay.bay2lnum(bay)
        if assume_usr_active:
            usr_state = drived.USR_ACTIVE
        else:
            # Also assume USR_ACTIVE if drive_d didn't know about this bay
            usr_state = usr_states.get(bay, drived.USR_ACTIVE)
        if (lnum > -1 and active == 1 and softfail == 0 and dbay.isHdd(bay)
            # XXX Significant rework required for mixed sector sizes
            and dbay.bay2blocklen(bay) == bootdisk.BSIZE
            # XXX Significant rework required for SEDs
            and not dbay.isSelfEncryptingDrive(bay)
            # Don't disrupt a firmware update in progress
            and usr_state is not drived.USR_FW_UPDATE
            # Sufficient free space or already in use for var
            and (var_count > 0 or free_space(geom))):
            eligible_geoms.append(geom)
    exclude_geoms = []
    if opts.exclude:
        exclude_drives = [ d.strip() for d in opts.exclude.split(',') ]
        for drive in exclude_drives:
            bay = dbay.any2bay(drive)
            if bay < 0:
                log_print_exit(msg_lvl=syslog.LOG_ERR,
                               msg='%s is not a valid drive to exclude' % drive)
            else:
                bay_geom = None
                for geom in bootdisk.mesh.partition_geoms:
                    if dbay.dev2bay(geom.name) == bay:
                        bay_geom = geom
                        break
                if bay_geom:
                    exclude_geoms.append(bay_geom)
    include_geoms = []
    if opts.include:
        include_drives = [ d.strip() for d in opts.include.split(',') ]
        for drive in include_drives:
            bay = dbay.any2bay(drive)
            if bay < 0:
                log_print_exit(msg_lvl=syslog.LOG_ERR,
                               msg='%s is not a valid drive to include' % drive)
            else:
                bay_geom = None
                for geom in eligible_geoms:
                    if dbay.dev2bay(geom.name) == bay:
                        bay_geom = geom
                        break
                if bay_geom:
                    if bay_geom in exclude_geoms:
                        log_print_exit(msg_lvl=syslog.LOG_ERR,
                                       msg='Excluding %s since it was also on '
                                       'the list of drives to exclude' % drive)
                    else:
                        include_geoms.append(bay_geom)
                else:
                    log_print_exit(msg_lvl=syslog.LOG_ERR, msg='%s is not an '
                                   'eligible drive to include' % drive)
    include_geoms += [ g for g in eligible_geoms
                       if g not in include_geoms and g not in exclude_geoms ]
    return (include_geoms, exclude_geoms)

def verify():
    ''' Return True if mirrors are healthy, False if not '''
    reqs = [ r for r in bootdisk.bootdisk_reqs if r.label in VAR_LKGS ]
    bootdisk.mesh.refresh()
    for req in reqs:
        try:
            mg = bootdisk.mesh.mirror_geom(req.label)
        except KeyError:
            log_print_exit(msg_lvl=syslog.LOG_ERR,
                           msg='Failed to find %s mirror geom' % req.label)
            return False
        if len(mg.consumers) < mg.Components:
            log_print_exit(msg_lvl=syslog.LOG_ERR,
                           msg='verify: missing components')
            return False
        if len(mg.consumers) < req.n:
            log_print_exit(msg_lvl=syslog.LOG_ERR,
                           msg='verify: missing requirements')
            return False
        if mg.State != 'COMPLETE':
            log_print_exit(msg_lvl=syslog.LOG_ERR,
                           msg='verify: not complete')
            return False
        for mc in mg.consumers:
            if mc.State != 'ACTIVE':
                log_print_exit(msg_lvl=syslog.LOG_ERR,
                               msg='verify: not active')
                return False
    return True

def free_space(pg):
    ''' Find or make enough free space by shrinking scratch '''
    req_blocks = bootdisk.var_req.get('size') / bootdisk.BSIZE
    for pp in pg.providers:
        if pp.label != 'scratch':
            continue
        try:
            scratch_blocks = int(pp.length) / bootdisk.BSIZE
        except ValueError:
            return False
        for space in pg.free_spaces:
            free_start = space[0]
            free_blocks = space[1]
            if free_start != pp.start + scratch_blocks:
                continue
            if free_blocks >= req_blocks:
                return True
            # Only resize scratch if the majority remains intact
            resize_blocks = scratch_blocks - (req_blocks - free_blocks)
            if round(float(resize_blocks) / float(scratch_blocks)) < 1.0:
                return False
            # XXX Rework this if/when isi.sys.geom can resize
            # XXX Also rework for patching 7.x where resize doesn't exist
            cmd = ('/sbin/gpart resize -i %d -s %d %s' %
                   (pp.index, resize_blocks, pg.name))
            err, out = procs.get_cmd_output(cmd, raw=True)
            bootdisk.mesh.refresh()
            if err:
                log_print_exit(msg_lvl=syslog.LOG_ERR, msg=out)
                return False
            return True
    return False

def migrate(opts, include_geoms, exclude_geoms):
    ''' Migrate var from bootdisks to storage drives '''
    all_storage_geoms = [ pp.geom for pp in bootdisk.mesh.partition_providers
                          if pp.label in VAR_LKGS
                          and pp.name.startswith('da') ]
    reqs = [ r for r in bootdisk.bootdisk_reqs if r.label in VAR_LKGS ]
    for req in reqs:
        try:
            mg = bootdisk.mesh.mirror_geom(req.label)
        except KeyError:
            log_print_exit(msg_lvl=syslog.LOG_ERR,
                           msg='Failed to find %s mirror geom' % req.label)
            continue
        storage_geoms = [ mc.provider.geom for mc in mg.consumers
                          if mc.provider.name.startswith('da')
                          and mc.provider.geom not in exclude_geoms ]
        exclude_providers = [ mc.provider for mc in mg.consumers
                              if mc.provider.geom not in storage_geoms ]
        if len(storage_geoms) < opts.copies:
            orig_req_n = req.n
            req.n = len(exclude_providers) + opts.copies
            add_geoms = []
            while len(add_geoms) < opts.copies - len(storage_geoms):
                try:
                    pg = include_geoms.pop(0)
                except IndexError:
                    log_print_exit(msg_lvl=syslog.LOG_ERR,
                                   msg='Exhausted available drives',
                                   exit_val=1)
                    break
                if pg in storage_geoms or pg in all_storage_geoms:
                    continue
                add_geoms.append(pg)
            log_print_exit(msg='Adding %s to %s' %
                           ([ g.name for g in add_geoms ], mg.name))
            # XXX Assumption: isi.sys.geom.GPTPartitionGeom.find_free_space()
            #                 uses first fit algorighm
            bootdisk.mirror_additions(req, alt_part_geoms=add_geoms)
            bootdisk.mesh.refresh()
            req.n = orig_req_n
            sync = bootdisk.verify_boot_device_synced()
            if sync:
                log_print_exit(msg_lvl=syslog.LOG_ERR,
                               msg='Failed to sync %s' % mg.name)
        else:
            log_print_exit(msg='No drives added to %s, %d copies already '
                           'exist' % (mg.name, opts.copies))
        if len(exclude_providers) > 0:
            log_print_exit(msg='Removing %s from %s' %
                           ([ p.name for p in exclude_providers ], mg.name))
            bootdisk.remove_providers(mg, exclude_providers)
        mg.forget()
    mirror_providers = []
    for mg in bootdisk.mesh.mirror_geoms:
        if mg.name in VAR_LKGS:
            mirror_providers += [ mc.provider for mc in mg.consumers ]
    exclude_providers = [ pp for pp in bootdisk.mesh.partition_providers
                          if pp.label in VAR_LKGS
                          and pp not in mirror_providers ]
    for provider in exclude_providers:
        log_print_exit(msg='Deleting %s' % provider.name)
        provider.delete()

def undo_migration(cc, ls, local_settings, ifs_ready):
    ''' Migrate var from storage drives to bootdisks '''
    bootdisk.undo_var_migration()
    if local_settings.get('enabled'):
        if ifs_ready:
            local_settings['enabled'] = False
            local_settings['hours'] = 0
            local_settings['timestamp'] = 0
            local_settings['blocks'] = 0
            ls['boot_flash_write_rate'] = local_settings
            cc.set_local_settings(ls)
        else:
            for sub in ('enabled=false', 'hours=0', 'timestamp=0', 'blocks=0'):
                cmd = LOCAL_GC_CMD % sub
                procs.get_cmd_output(cmd)

def create_placeholders():
    ''' Reserve the space where var used to be for future reverse migration '''
    reqs = [ bootdisk.BootdiskRequirement(geom.GPT_ENT_TYPE_FREEBSD_UFS, label,
             bootdisk.var_req['size'], bootdisk.var_req['num'], gmirror=False)
             for label in bootdisk.VAR_MIGRATION_PLACEHOLDERS ]
    for req in reqs:
        bootdisk.partition_additions(req)

def main():
    ''' Main function (handle gconfig settings in this context) '''

    syslog.openlog('%s[%d]' % (os.path.basename(__file__), os.getpid()))

    # Platform check
    if bootstrap.getMotherboardType() not in bootdisk.VAR_MIGRATION_PLATFORMS:
        log_print_exit(msg='This platform is not supported by this script',
                       exit_val=0)

    # Parse options and sanity check
    parser = option_parser_setup()
    opts, args = parser.parse_args()
    opt_count = (opts.migrate, opts.rebalance, opts.undo,
                 opts.disable, opts.enable, opts.status).count(True)
    if opt_count > 1:
        log_print_exit(msg='Conflicting options')
        parser.print_help()
        log_print_exit(exit_val=0)
    opt_count += sum([opts.threshold != None,
                      opts.duration != None,
                      opts.copies != None])
    if opt_count < 1:
        log_print_exit(msg='No option specified')
        parser.print_help()
        log_print_exit(exit_val=0)

    # Force multiple instances to wait using lock file
    lockfd = open('/tmp/%s.lock' % os.path.basename(__file__), 'w')
    fcntl.lockf(lockfd, fcntl.LOCK_EX)
    bootdisk.mesh.refresh()

    # Check for quorum (required to get settings off /ifs)
    ifs_ready = False
    cc = ls = None
    if misc.ifs_ready():
        ifs_ready = True
        # Get global and local settings
        cc = ccapi.CelogConfigAPI()
        ms = cc.get_monitor_settings()
        global_settings = ms.get('boot_flash_write_rate')
        ls = cc.get_local_settings()
        local_settings = ls.get('boot_flash_write_rate')
    elif opts.rebalance or opts.undo:
        # Get the local enable the hard way when /ifs isn't available
        local_settings = {}
        cmd = LOCAL_GC_CMD % 'enabled'
        err, out = procs.get_cmd_output(cmd, raw=True)
        if 'true' in out:
            local_settings['enabled'] = True
    else:
        log_print_exit(msg='Cluster does not have quorum', exit_val=0)

    # Event threshold and duration options
    if ifs_ready and opts.threshold:
        if opts.threshold == global_settings.get('threshold'):
            log_print_exit(msg_lvl=syslog.LOG_INFO,
                           msg='Event threshold is already set to %d GB/day' %
                           global_settings.get('threshold'))
            opt_count -= 1
        elif opts.threshold > 0:
            global_settings['threshold'] = opts.threshold
            ms['boot_flash_write_rate'] = global_settings
            cc.set_monitor_settings(ms)
            log_print_exit(msg='Event threshold set to %d GB/day' %
                           global_settings.get('threshold'))
            opt_count -= 1
        else:
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Event threshold '
                           'must be greater than 0 GB/day', exit_val=1)
    if ifs_ready and opts.duration:
        if opts.duration == global_settings.get('duration'):
            log_print_exit(msg_lvl=syslog.LOG_INFO,
                           msg='Event duration is already set to %d hours' %
                           global_settings.get('duration'))
            opt_count -= 1
        elif opts.duration > 0:
            global_settings['duration'] = opts.duration
            ms['boot_flash_write_rate'] = global_settings
            cc.set_monitor_settings(ms)
            log_print_exit(msg='Event duration set to %d hours' %
                           global_settings.get('duration'))
            opt_count -= 1
        else:
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Event duration '
                           'must be greater than 0 hours', exit_val=1)

    # Copies option
    min_copies = bootdisk.var_req.get('num')
    if ifs_ready and opts.copies:
        if opts.copies == global_settings.get('var_copies'):
            log_print_exit(msg_lvl=syslog.LOG_INFO, msg='Copies is already '
                           'set to %d' % global_settings.get('var_copies'))
            opt_count -= 1
        elif opts.copies >= min_copies:
            global_settings['var_copies'] = opts.copies
            ms['boot_flash_write_rate'] = global_settings
            cc.set_monitor_settings(ms)
            log_print_exit(msg='Copies set to %d' %
                           global_settings.get('var_copies'))
            opt_count -= 1
        else:
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Copies must '
                           'be %d or greater' % min_copies, exit_val=1)
    elif ifs_ready:
        opts.copies = global_settings.get('var_copies')
    else:
        opts.copies = min_copies

    # Remaining options are mutually exclusive (enforced by sanity check above)
    # Exit now if not performing any of the mutually exclusive options
    if opt_count < 1:
        log_print_exit(exit_val=0)

    # Status option
    if ifs_ready and opts.status:
        status(global_settings)
        log_print_exit(exit_val=0)

    # Enable option
    if ifs_ready and opts.enable:
        if global_settings.get('enabled'):
            log_print_exit(msg_lvl=syslog.LOG_INFO, msg='Var migration '
                           'is already glabally enabled', exit_val=0)
        global_settings['enabled'] = True
        ms['boot_flash_write_rate'] = global_settings
        cc.set_monitor_settings(ms)
        log_print_exit(exit_val=0)

    # Disable option
    if ifs_ready and opts.disable:
        if not global_settings.get('enabled'):
            log_print_exit(msg_lvl=syslog.LOG_INFO, msg='Var migration '
                           'is already glabally disabled', exit_val=0)
        global_settings['enabled'] = False
        ms['boot_flash_write_rate'] = global_settings
        cc.set_monitor_settings(ms)
        # End any outstanding events of this type
        cdb = query.Query()
        events = cdb.get_events(eventids=[eventids.eventid_map[
                                "SYS_DISK_BOOT_FLASH_WRITE_RATE"]])
        for event in events:
            event.end()
        # Reset the local counters
        local_settings['hours'] = 0
        local_settings['timestamp'] = 0
        local_settings['blocks'] = 0
        ls['boot_flash_write_rate'] = local_settings
        cc.set_local_settings(ls)
        log_print_exit(exit_val=0)

    # Undo option
    if opts.undo:
        log_print_exit(msg='Migrating var back to boot flash drives')
        undo_migration(cc, ls, local_settings, ifs_ready)
        log_print_exit(exit_val=0)

    (include_geoms, exclude_geoms) = get_geoms(opts)
    if len(include_geoms) == 0:
        if local_settings.get('enabled'):
            exit_val = 1
        else:
            exit_val = 0
        log_print_exit(msg_lvl=syslog.LOG_ERR, msg='No usable storage '
                       'drives found for var migration', exit_val=exit_val)

    # Migrate option (requires global enable, provides local enable)
    if ifs_ready and opts.migrate:
        if not global_settings.get('enabled'):
            log_print_exit(msg='Var migration is globally disabled', exit_val=0)
        if len(include_geoms) < opts.copies * len(bootdisk.lkg_suffixes):
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Not enough drives '
                           'to fulfill %d copies' % opts.copies, exit_val=1)
        if not verify():
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Mirrors are not in '
                           'a healthy state for var migration', exit_val=1)
        migrate(opts, include_geoms, exclude_geoms)
        if not verify():
            undo_migration(cc, ls, local_settings, ifs_ready)
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Mirrors were '
                           'not healthy after var migration', exit_val=1)
        create_placeholders()
        if not local_settings.get('enabled'):
            local_settings['enabled'] = True
            ls['boot_flash_write_rate'] = local_settings
            cc.set_local_settings(ls)
        log_print_exit(exit_val=0)

    # Rebalance option (same as migrate, but requires local enable)
    if opts.rebalance:
        if not local_settings.get('enabled'):
            log_print_exit(msg='Var migration is locally disabled', exit_val=0)
        if len(include_geoms) < opts.copies * len(bootdisk.lkg_suffixes):
            undo_migration(cc, ls, local_settings, ifs_ready)
            # Exit 0 so drive_d can transition drives from smartfail to replace
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Not enough drives '
                           'to fulfill %d copies, var has been migrated back '
                           'to boot flash drives' % opts.copies, exit_val=0)
        migrate(opts, include_geoms, exclude_geoms)
        if not verify():
            undo_migration(cc, ls, local_settings, ifs_ready)
            # Exit 0 so drive_d can transition drives from smartfail to replace
            log_print_exit(msg_lvl=syslog.LOG_ERR, msg='Mirrors were not '
                           'healthy after rebalance, var has been migrated '
                           'back to boot flash drives', exit_val=0)
        log_print_exit(exit_val=0)

    log_print_exit(exit_val=0)

if __name__ == '__main__':
    main()
