#! /usr/bin/env python

# Optimize imports for this user-facing tool, as best we can.
# For some "long" import time modules, import only when needed in funcs.

import os
import re
import tempfile

REVISION = filter(str.isdigit, "\$Revision: 485457 $")
VERSION = 3
TITLE = "VPD (Vital Product Data) Tool"

_fdbg = False

import sys
import time
import traceback
import subprocess

import isi.hw.hal as hal
import isi.hw.ipmi as ipmi

import isi.hw.check.lib.args as argslib
import isi.hw.check.lib.filenames as files

FLASH_FILESYSTEM_EEPROM_VERSION = 3

# default files to be saved to FLASH from CTO files
DEFAULT_FLASH_FILESYSTEM_CTO_FILES = [
"/mfg/asbuilts/asBuilt.xml\n",
"/mfg/probeconfigs/probe_config.gc\n",
"/mfg/probeconfigs/probe_config.0.gc\n",
"/mfg/probeconfigs/probe_config.1.gc\n",
"/mfg/receipts/receipt.xml\n",
"/mfg/softAVLs/softAVL.xml\n"]

ROOT_DIR = os.path.sep
MFG_DIR = '/mfg'
MFG_PARTITION = os.path.join(ROOT_DIR, 'dev', 'mirror', 'mfg')

_vpdfile = files.node_vpd_data_file()
_debug = False
_verbose = True # Does double duty. Can be nonzero integer.
_force = False
_single = False
_devid = None

_max_attempts = 3

def fdbg(*arg):
    if _fdbg:
        msg = ""
        for m in arg:
            msg = msg.join(m)
        print msg

def verbose(msg):
    if _verbose:
        print msg

def error(msg, err=None):
    print msg
    if err:
        print err

def opt2int(opt, base=16):
    r, x = 0, None
    try:
        x = int(opt)
    except ValueError:
        try:
            x = int(opt, base)
            fdbg("opt2int - converted %s to int" % opt)
        except ValueError:
            r = 1
            error('opt2int unable to convert arg "%s" to int' % opt)
    return r, x

def fs_mounted(fsname=None):
    rc, err, out = system_command_output('mount | grep %s' % fsname)
    if rc or err:
        # fs does not appear to be mounted
        mounted = False
    else:
        mounted = True
    return mounted

def unmount_fs(fsname=None):
    """Attempts unmount, returns None on success, error msg on fail"""
    msg = None
    if fsname is not None:
        if not fs_mounted(fsname):
            msg = '%s is not mounted' % fsname
        else:
            rc, err, output = system_command_output('umount %s' % fsname)
            if rc or err:
                # Try forcing
                rc, err, output = system_command_output('umount -f %s' % fsname)
                if rc or err:
                    msg = 'Unable to unmount %s' % fsname
            else:
                msg = '%s unmounted' % fsname
    else:
        msg = 'Filesystem name not specified'
    return msg

def init(vpd, options):
    # optionally have a flag to force no prompt;
    verbose("Initializing %s data..." % vpd.name())
    err = vpd.eeprom_init(force=_force)
    if err:
        error("Unable to initialize %s VPD data:" % vpd.name(), err)
    r = err and 1 or 0
    return r

def initrev(vpd, options):
    # optionally have a flag to force no prompt;
    r, version = opt2int(options['version'], base=10)
    if not r:
        verbose("Initializing %s data for rev %d..." % (vpd.name(), version))
        err = vpd.eeprom_init(rev=version, force=_force)
        if err:
            error("Unable to initialize %s VPD data:" % vpd.name(), err)
        r = err and 1 or 0
    return r

def _check(vpd, options):
    verbose("Checking %s data..." % vpd.name())
    # we can pass specific checks to this func w/ keyword checks=[...]
    err, temp = vpd.check_data()
    if err:
        error("%s header/data invalid: Consider re-init?:" % vpd.name(), err)
    else:
        verbose("%s header and data appear valid!" % vpd.name())
    r = err and 1 or 0
    return r

def check(vpd, options):
    if isinstance(vpd, list):
        vpdlist = vpd
    else:
        vpdlist = [vpd]
    return reduce(lambda a, b: a or b,
                  [_check(v, options) for v in vpdlist], 0)

# Consider: making _vpdfile an option.
# This allows syncing EEPROM vpd data with a user-specified non-default
# file, e.g., a /tmp/ located file, for integration with isi_dongle_sync,
# for single call with isi_dongle_sync "pull" :
#    {[dongle], [eeprom]} --> {temp} --> {<legacy files>, <vpd file>}
# This requires careful consideration and integration of the current
# coding paradigm in (cmds_list,cmds_desc) methods, 'func', and the
# generic calls:
# argfunc = cmds_desc[cmd]['argfunc'] --> func_options
# f = cmds_desc[cmd]['func'] = sync(), result = f(vpd, func_options)

def sync(vpd, options):
    verbose("Syncing %s data with local files..." % vpd.name())
    err = vpd.update('sync')
    if err:
        error("%s sync with local files failed:" % vpd.name(), err)
    r = err and 1 or 0
    return r

def push(vpd, options):
    verbose("Pushing VPD data to %s from local files..." % vpd.name())
    err = vpd.update('push')
    if err:
        error("%s push with local files failed:" % vpd.name(), err)
    r = err and 1 or 0
    return r

def pull(vpd, options):
    verbose("Pulling VPD data from %s to local files..." % vpd.name())
    err = vpd.update('pull')
    if err:
        error("%s pull with local files failed:" % vpd.name(), err)
    r = err and 1 or 0
    return r

def _dump(vpd, options):
    verbose("Reading data from %s..." % vpd.name())
    r = 0
    err, lines = vpd.dump_data(options={'single':_single})
    if err:
        error("Error reading %s data:" % vpd.name(), err)
        r = 1
        if not _debug:
            return r
    if lines:
        if err and _debug:
            print("Displaying %s dump data for debug purposes - "
                  "data values may be incorrect" % vpd.name()) # Non-verbose
        verbose('%s data dump:' % vpd.name())
        for line in lines:
            print line # Non-verbose
    return r

def dump(vpd, options):
    if isinstance(vpd, list):
        vpdlist = vpd
    else:
        vpdlist = [vpd]
    return reduce(lambda a, b: a or b,
                  [_dump(v, options) for v in vpdlist], 0)

# Optional args: offset, length
def _dumpraw(vpd, options):
    verbose("Reading (raw) data from %s..." % vpd.name())
    try:
        opt = options['off']
        r, offset = opt2int(opt)
        if r:
            return r
    except KeyError:
        offset = 0
    try:
        opt = options['len']
        r, length = opt2int(opt)
        if r:
            return r
    except KeyError:
        length = None # Default to entire EEPROM
    err, lines = vpd.dump_raw(offset, length, block_protect=False)
    if err:
        error("Error reading %s (raw) data:" % vpd.name(), err)
    else:
        verbose('%s (raw) data dump:' % vpd.name())
        for line in lines:
            print line # Non-verbose
    r = err and 1 or 0
    return r

def dumpraw(vpd, options):
    if isinstance(vpd, list):
        vpdlist = vpd
    else:
        vpdlist = [vpd]
    return reduce(lambda a, b: a or b,
                  [_dumpraw(v, options) for v in vpdlist], 0)

def dumphdr(vpd, options):
    verbose("Reading header data from %s..." % vpd.name())
    raw = _debug
    err, lines = vpd.dump_header(raw=raw)
    if err:
        error("Error reading %s header:" % vpd.name(), err)
    else:
        verbose('%s header dump:' % vpd.name())
        if raw:
            err, temp = vpd.check_data()
            if err:
                verbose('NOTE: HEADER IS NOT VALID! Debug mode, forcing dump.')
                verbose('%s' % err)
                err = None
        if isinstance(lines, list ):
            for line in lines:
                print line # Non-verbose
    r = err and 1 or 0
    return r

def _dumpresume(vpd, options):
    global _verbose
    verbose("Reading data from %s..." % vpd.name())
    err, lines = vpd.eeprom.dump_data(options={'verbose': _verbose})
    if err:
        error("Error reading %s data:" % vpd.name(), err)
    else:
        verbose("%s (EEPROM-only) data dump:" % vpd.name())
        verbose("(*'ed entries are Required EMC Fields)")
        for line in lines:
            print line # Non-verbose
    r = err and 1 or 0
    return r

def dumpresume(vpd, options):
    if isinstance(vpd, list):
        vpdlist = vpd
    else:
        vpdlist = [vpd]
    return reduce(lambda a, b: a or b,
                  [_dumpresume(v, options) for v in vpdlist], 0)

def _read(vpd, keys):
    verbose("Reading data from %s: keys=%s" % (vpd.name(), keys))
    if not len(keys):
        return 0 # No data to read
    r = 0
    err, data = vpd.read_data()
    if err:
        error("Failure %s Read (data):" % vpd.name(), err)
        r = 1
        if not _debug:
            return r
    if data:
        if err and _debug:
            print("Displaying %s read data for debug purposes - "
                  "data values may be incorrect" % vpd.name()) # Non-verbose
            err = None
        read = {}
        for key in keys:
            try:
                value = data[key]
            except KeyError:
                err = ("Failure %s Read: one or more keys not found" %
                       vpd.name())
                value = None
            read[key] = value
        width = max(map(lambda key:len(key), read.keys()))
        errval = "Error: key not found"
        for key in keys:
            found = key in read and not read[key] is None
            value = not found and errval or read[key]
            if _single:
                value = found and len(value) and value[0] or ''
            print "%-*s = %s" % (width, key, value) # Non-verbose
        if err:
            error(err)
            r = 1
    return r

# Requires arg: key
# This requires reading all data with vpd.read_data,
# then parsing results... unless we add a new method to eeprom?
def read(vpd, options):
    if isinstance(vpd, list):
        vpdlist = vpd
    else:
        vpdlist = [vpd]
    keys = [ options['key'] ]
    return reduce(lambda a, b: a or b,
                  [_read(v, keys) for v in vpdlist], 0)

# Requires args: <count> + keys
# This requires reading all data with vpd.read_data,
# then parsing results... unless we add a new method to eeprom?
def read_n(vpd, options):
    if isinstance(vpd, list):
        vpdlist = vpd
    else:
        vpdlist = [vpd]
    r, count = opt2int(options['len'], base=10)
    if r:
        return r
    # keys should be a list of keys per len_keys_get_args
    keys = options['[keys]']
    if None in [count,keys]:
        return 1
    return reduce(lambda a, b: a or b,
                  [_read(v, keys) for v in vpdlist], 0)

def _delete(vpd, keys):
    verbose("Deleting data from %s: keys=%s" % (vpd.name(), keys))
    if not len(keys):
        return 0 # No data to delete
    err = vpd.delete(keys)
    if err:
        error("Failure %s Delete:" % vpd.name(), err)
    r = err and 1 or 0
    return r

# Requires arg: key
def delete(vpd, options):
    keys = [ options['key'] ]
    r = _delete(vpd, keys)
    return r

# Requires args: <count> + keys
def delete_n(vpd, options):
    r, count = opt2int(options['len'], base=10)
    if r:
        return r
    # keys should be a list of keys per len_keys_get_args
    keys = options['[keys]']
    if None in [count,keys]:
        return 1
    r = _delete(vpd, keys)
    return r

def _write(vpd, data):
    verbose("Writing data to %s: %s" % (vpd.name(), data))
    if not len(data):
        return 0 # No data to write
    err, temp_info = vpd.sync_vpd(new_vpd_data=data, priority='eeprom_only')
    if err:
        error("Failure VPD Update (data=%s):" % data, err)
    else:
        err = vpd.eeprom_update()
        if err:
            error("Failure %s Write (data=%s):" % (vpd.name(), data), err)
    r = err and 1 or 0
    return r

# Requires arg: key:value dict pair
def write(vpd, options):
    userdata = { options['key'] : options['value'] }
    r = _write(vpd, userdata)
    return r

# Requires args: <count> + key:value dict pairs
def write_n(vpd, options):
    r, count = opt2int(options['len'], base=10)
    if r:
        return r
    # userdata should be a dict of key:value pairs per len_key_value_get_args
    userdata = options['[key value]']
    if None in [count,userdata]:
        return 1
    r = _write(vpd, userdata)
    return r

# Requires arg: offset, length, list
def writeraw(vpd, options):
    r, offset = opt2int(options['off'])
    if r:
        return r
    r, length = opt2int(options['len'])
    if r:
        return r

    # data should already be a list of ints per off_len_data_get_args
    data = options['data']
    if None in [offset,length,data]:
        return 1
    verbose("Writing data %s to %s at offset 0x%x..." %
            (data, vpd.name(), offset))
    err = vpd.write_raw(offset, data)
    if err:
        error("Failure %s Write (data=%s):" % (vpd.name(), data), err)
    r = err and 1 or 0
    return r

def len_keys_get_args(usercmds, cmdargs):
    # Last opt '[keys]' is variable len, >= 1, of keys
    valid, options = True, {}
    numarg = len(cmdargs) - 1
    userargs = usercmds[:numarg]
    usercmds = usercmds[numarg:]
    if len(userargs) != numarg:
        valid = False
    if valid:
        options = dict(zip(cmdargs,userargs))
        r, datalen = opt2int(options['len'], 10)
        if r:
            valid = False
        elif datalen is None:
            valid = False
    if valid:
        dataargs = usercmds[:datalen]
        usercmds = usercmds[datalen:]
        if len(dataargs) != datalen:
            valid = False
    if valid:
        data = dataargs
        options['[keys]'] = data
    return valid, options, usercmds

def len_key_value_get_args(usercmds, cmdargs):
    # Last opt '[key value]' is variable len, >= 1, of key:value pairs
    valid, options = True, {}
    numarg = len(cmdargs) - 1
    userargs = usercmds[:numarg]
    usercmds = usercmds[numarg:]
    if len(userargs) != numarg:
        valid = False
    if valid:
        options = dict(zip(cmdargs,userargs))
        r, datalen = opt2int(options['len'], 10)
        if r:
            valid = False
        elif datalen is None:
            valid = False
    if valid:
        datalen *= 2 # key,value
        dataargs = usercmds[:datalen]
        usercmds = usercmds[datalen:]
        if len(dataargs) != datalen:
            valid = False
    if valid:
        data = [[dataargs[x] for x in range(y,datalen,2)] for y in range(2)]
        data = dict(zip(*data))
        options['[key value]'] = data
    return valid, options, usercmds

def off_len_data_get_args(usercmds, cmdargs):
    # Last opt 'data' is variable len, >= 1
    valid, options = True, {}
    numarg = len(cmdargs) - 1
    userargs = usercmds[:numarg]
    usercmds = usercmds[numarg:]
    if len(userargs) != numarg:
        valid = False
    if valid:
        options = dict(zip(cmdargs,userargs))
        r, datalen = opt2int(options['len'])
        if r:
            valid = False
        elif datalen is None:
            valid = False
    if valid:
        dataargs = usercmds[:datalen]
        usercmds = usercmds[datalen:]
        if len(dataargs) != datalen:
            valid = False
    if valid:
        data = []
        for arg in dataargs:
            r, d = opt2int(arg)
            if r:
                valid = False
                break
            data.append(d)
    if valid:
        invalid = filter(lambda x: not isinstance(x,int), data)
        if len(invalid):
            valid = False
    if valid:
        options['data'] = data
    return valid, options, usercmds

def generic_get_args(usercmds, cmdargs):
    options = None
    valid = True
    numarg = len(cmdargs)
    userargs = usercmds[:numarg]
    usercmds = usercmds[numarg:]
    if len(userargs) != numarg:
        valid = False
    else:
        options = dict(zip(cmdargs,userargs))
    return valid, options, usercmds

def dumppbb():
    import isi.hw.version as hwver
    import isi.hw.check.lib.eeprom as eeprom
    print 'Power Bridge Board EEPROM data dump:'
    result = 0
    fruid = 0
    bus = hal.getPbbVpdEepromBus()
    addr = hal.getPbbVpdEepromAddr()
    size = hal.getPbbVpdEepromSize()
    if hal.getChassisType(0) in [ hwver.CHASSIS_ISI36BAY_V3 ] :
        fruid = ipmi.FRUDEVID_PBB
    pbb_eeprom = eeprom.Eeprom8Bit(bus=bus, addr=addr, size=size, fruid=fruid)
    err, info = pbb_eeprom.data_dump()
    if err:
        return 1
    for (key_width, name, value) in info:
        print '%-*s : %s' % (key_width, name, value)
    return result

opts_desc = [
    ('-z, --fdbg',     'Debug flash mode'),
    ('-s, --single',   'Single value dumps (no history)'),
    ('-f, --force',    'Force prompting (no confirms)'),
    ('-d, --debug',    'Debug mode'),
    ('-q, --quiet',    'Quiet mode'),
    ('-h, --help',     'Show this text'),
    ]

if hal.supportsResumeEeprom():
    opts_desc.extend([
    ('--devid=DEVID',  'Device Id (For Resume EEPROMs)'),
    ])

cmds_list = [
    ('init',
    { 'func': init,
      'title': 'EEPROM Init to latest version (destructive)',
      }),
    ('initrev',
    { 'func': initrev,
      'title': 'EEPROM Init with given version (destructive)',
      'args' : ['version'],
      }),
    ('check',
    { 'func': check,
      'title': 'EEPROM Data Check (validate header, data)',
      }),
    ('push',
    { 'func': push,
      'title': 'Push data - to EEPROM from local VPD files',
      }),
    ('pull',
    { 'func': pull,
      'title': 'Pull data - from EEPROM to local VPD files',
      }),
    ('sync',
    { 'func': sync,
      'title': 'Sync data - EEPROM with local VPD files (uses timestamps)',
      }),
    ('dump',
    { 'func': dump,
      'title': 'Dump EEPROM - data, formatted',
      }),
    ('dumpresume',
    { 'func': dumpresume,
      'title': 'Dump EEPROM - Resume data, formatted',
      'resume': True,
      }),
    ('dumphdr',
    { 'func': dumphdr,
      'title': 'Dump EEPROM - header, formatted',
      }),
    ('dumpraw',
    { 'func': dumpraw,
      'title': 'Dump EEPROM - raw data, at given offset + length',
      'args': ['off','len'],
      'raw':True,
      }),
    ('dumpall',
    { 'func': dumpraw,
      'title': 'Dump EEPROM - raw data, ALL (long!)',
      'raw':True,
      }),
    ('read',
    { 'func': read,
      'title': 'Read EEPROM - read given key (search)',
      'args': ['key'],
      }),
    ('readn',
    { 'func': read_n,
      'title': 'Read EEPROM - read "len" keys (search)',
      'args': ['len','[keys]'],
      'argfunc': len_keys_get_args,
      }),
    ('delete',
    { 'func': delete,
      'title': 'Write EEPROM - delete given key',
      'args': ['key'],
      }),
    ('deleten',
    { 'func': delete_n,
      'title': 'Write EEPROM - delete "len" keys',
      'args': ['len','[keys]'],
      'argfunc': len_keys_get_args,
      }),
    ('write',
    { 'func': write,
      'title': 'Write EEPROM - write given key:value pair',
      'args': ['key','value'],
      }),
    ('writen',
    { 'func': write_n,
      'title': 'Write EEPROM - write "len" key:value pairs',
      'args': ['len','[key value]'],
      'argfunc': len_key_value_get_args,
      }),
    ('writeraw',
    { 'func': writeraw,
      'title': 'Write EEPROM - write raw data to given offset',
      'args': ['off','len','data'],
      'argfunc': off_len_data_get_args,
      'raw':True,
      }),
    ]

# Special - multi-vpd list for Resume EEPROM VPDs
multi_devid_cmds_list = [
    'dump', 'dumpraw', 'dumpall', 'check', 'read', 'readn'
    ]

_node_has_flash = hal.supportsChassisVpdFlash()

_node_has_resume = hal.supportsResumeEeprom()

_group_flash_cmds = False # In "help" usage display listing

def get_devid(cmd):
    global _devid, multi_devid_cmds_list
    devid = _devid
    invalid_msg = "Invalid resume devid %s" % devid
    if devid is None:
        devid = hal.getResumeEepromDevid()
    if isinstance(devid, int):
        pass
    elif isinstance(devid, basestring):
        if devid.upper() == 'ALL':
            # Special - multi-vpd list for Resume EEPROM VPDs
            if cmd in multi_devid_cmds_list:
                devid = hal.getResumeEepromDevidList()
                if not isinstance(devid, list):
                    raise Exception(invalid_msg)
            else:
                raise Exception(invalid_msg)
        else:
            try:
                devid = int(devid)
            except:
                raise Exception(invalid_msg)
    else:
        raise Exception(invalid_msg)
    return devid

# get vpd for non-flash commands
vpd = None
def get_vpd(options, cmd=None):
    global vpd
    fdbg("flashtool - get_vpd")
    if vpd:
        return vpd
    try:
        import isi.hw.check.lib.vpd as vpdlib
        opts = options.get('vpd', {})
        if _node_has_resume:
            devid = get_devid(cmd)
            if isinstance(devid, list):
                vpd = []
                for id in devid:
                    opts['devid'] = id
                    vpd.append(vpdlib.get_vpd(options=opts))
            else:
                opts['devid'] = devid
                vpd = vpdlib.get_vpd(options=opts)
        else:
            # Make sure devid was not specified on command line -
            # does not apply to non-Resume EEPROMs, something wrong.
            global _devid
            if _devid is not None:
                raise Exception('Devid %s specified, not supported' % _devid)
            vpd = vpdlib.get_vpd(options=opts)
    except Exception as e:
        error('%s: %s' % (type(e), e))
        error('Could not init EEPROM VPD object')
        # Fatal error
        sys.exit(1)
    return vpd

# get vpd for flash commands
vpd_ef = None
def get_vpd_ef(options):
    global vpd_ef
    fdbg("flashtool - get_vpd_ef")
    if vpd_ef:
        return vpd_ef
    try:
        import isi.hw.check.lib.vpd as vpdlib
        opts = options.get('vpd', {})
        vpd_ef = vpdlib.get_flash_vpd(options=opts)
    except Exception as e:
        error('%s: %s' % (type(e), e))
        error('Could not init FLASH VPD object')
        # Fatal error
        sys.exit(1)
    fdbg("flashtool - call vpd_flash_init")
    vpd_ef.vpd_flash_init()
    fdbg("flashtool - called vpd_flash_init")
    return vpd_ef

flash_file_system = None
def get_flash_file_system(options):
    global flash_file_system
    fdbg("FS - get_flash_file_system options=%s" % options)
    if flash_file_system:
        fdbg("FS - get_flash_file_system have it return")
        return flash_file_system
    attempts = 0
    while attempts < _max_attempts:
        try:
            attempts = attempts + 1
            fdbg("FS - get_flash_file_system set flash_file_system")
            flash_file_system = CMCFlashFileSystem(options)
            if flash_file_system:
                break;
        except Exception as e:
            flash_file_system = None
            continue;
    if attempts == _max_attempts and flash_file_system is None:
            error('%s: %s' % (type(e), e))
            error('Could not init Flash File System object')
            # Fatal error
            sys.exit(1)

    fdbg("FS - get_flash_file_system done")
    return flash_file_system


# ##################### #
# FLASH/EEPROM COMMANDS #
# ##################### #

def flash_init(options):
    fdbg("flash_init")
    # optionally have a flag to force no prompt;
    verbose("Initializing FLASH data...")
    vpd = get_vpd_ef(options)
    options['vpd'].update({'init': True})
    # force use of FLASH_FILESYSTEM_EEPROM_VERSION for flash EEPROM header
    err = vpd.eeprom_init(rev=FLASH_FILESYSTEM_EEPROM_VERSION, force=_force)
    if err:
        error("Unable to initialize FLASH VPD data", err)
    else:
        err = delete_file(files.node_flash_meta_data_file())
        if err:
            error("Unable to remove metadata file %s" %
                    files.node_flash_meta_data_file())
        else:
            err = init_f(options)
            if err:
                error("Unable to initialize FLASH VPD data", err)
    r = err and 1 or 0
    fdbg("flashtool - flash_init exit r=%s" % r)
    return r

def flash_initrev(options):
    # optionally have a flag to force no prompt;
    fdbg("flashtool - flash_initrev")
    r, version = opt2int(options['version'], base=10)
    if not r:
        verbose("Initializing FLASH data for rev %d..." % version)
        vpd = get_vpd_ef(options)
        options['vpd'].update({'init': True})
        # force use of FLASH_FILESYSTEM_EEPROM_VERSION for flash EEPROM header
        if version != FLASH_FILESYSTEM_EEPROM_VERSION:
            print("WARNING limited functionality with EEPROM header version %d:" % version)
            print("Recomend using header version %s" %
                    FLASH_FILESYSTEM_EEPROM_VERSION)
            err = vpd.eeprom_init(rev=version, force=_force)
            if err:
                error("Unable to initialize EEPROM for FLASH VPD metadata",
                       err)
            else:
                err = init_f(options)
                if err:
                    error("Unable to initialize FLASH VPD metadata", err)
        else:
            err = vpd.eeprom_init(rev=version, force=_force)
            if err:
                error("Unable to initialize EEPROM for FLASH VPD metadata",
                        err)
            else:
                err = init_f(options)
                if err:
                    error("Unable to initialize FLASH VPD metadata", err)
        r = err and 1 or 0
    return r

def flash_check(options):
    verbose("Checking FLASH metadata...")
    # we can pass specific checks to this func w/ keyword checks=[...]
    vpd = get_vpd_ef(options)
    err, temp = vpd.check_data()
    if err:
        error("FLASH header/metadata invalid: Consider re-init?:", err)
    else:
        verbose("EEPROM header and data appear valid!")
        err = check_f(options)
        if err:
            error("FLASH metadata invalid: Consider re-init?:", err)
    r = err and 1 or 0
    return r

def flash_initcto(options):
    rc = 0
    dfile = files.node_flash_data_file()
    verbose("Create default %s CTO file..." % dfile)
    try:
        with open(dfile, 'w') as f:
            for line in DEFAULT_FLASH_FILESYSTEM_CTO_FILES:
                f.write(line)
        verbose("Create %s Passed" % dfile)
    except ValueError:
        error('Unable to create %s' % dfile)
        rc = 1
    r = rc and 1 or 0
    return r

def flash_copy(options):
    verbose("Copying CTO files from %s to FLASH..." % MFG_DIR)
    vpd = get_vpd_ef(options)
    err = get_flash_file_system(options).copycto()
    r = err and 1 or 0
    return r

def flash_restore(options):
    verbose("Restoring CTO files from FLASH to %s..." % MFG_DIR)
    vpd = get_vpd_ef(options)
    restoremfg = 1
    err = get_flash_file_system(options).restorecto(restoremfg)
    r = err and 1 or 0
    return r

def flash_tmp(options):
    verbose("Restoring CTO files from FLASH to /tmp...")
    vpd = get_vpd_ef(options)
    err = get_flash_file_system(options).restorecto(restoremfg=0, md5=False)
    r = err and 1 or 0
    return r

def flash_tmp_md5(options):
    verbose("Restoring CTO MD5 files from FLASH to /tmp...")
    vpd = get_vpd_ef(options)
    err = get_flash_file_system(options).restorecto(restoremfg=0, md5=True)
    r = err and 1 or 0
    return r

def flash_push(options):
    verbose("Pushing FLASH metadata to EEPROM from local files...")
    vpd = get_vpd_ef(options)
    err = vpd.update('push')
    if err:
        error("FLASH metadata push with local files failed:", err)
    r = err and 1 or 0
    return r

def flash_pull(options):
    verbose("Pulling FLASH metadata from EEPROM to local files...")
    vpd = get_vpd_ef(options)
    err = vpd.update('pull')
    if err:
        error("FLASH metadata pull with local files failed:", err)
    r = err and 1 or 0
    return r

def flash_dump(options):
    verbose("Reading FLASH metadata header from EEPROM...")
    vpd = get_vpd_ef(options)
    err, lines = vpd.dump_data(options={'single':_single})
    if err:
        error("Error reading FLASH metadata header:", err)
    else:
        print('FLASH metadata dump:')
        for line in lines:
            print line # Non-verbose
        err = dump_f(options)
        if err:
            error("Error reading FLASH metadata:", err)
    r = err and 1 or 0
    return r

# Optional args: offset, length
def flash_dumprawhdr(options):
    verbose("Reading (raw) FLASH header data from EEPROM...")
    try:
        opt = options['off']
        r, offset = opt2int(opt)
        if r:
            return r
    except KeyError:
        offset = 0
    try:
        opt = options['len']
        r, length = opt2int(opt)
        if r:
            return r
    except KeyError:
        length = None # Default to entire EEPROM
    vpd = get_vpd_ef(options)
    # offset to start of our "flash" eeprom area
    base_size = hal.getChassisVpdEepromSize()
    if base_size == None:
        err = 1
    else:
        offset += base_size
        err, lines = vpd.dump_raw(offset, length, block_protect=False)
    if err:
        error("Error reading FLASH (raw) metadata from EEPROM:", err)
    else:
        verbose('FLASH (raw) metadata dump:')
        for line in lines:
            print line # Non-verbose
    r = err and 1 or 0
    return r

def flash_dumphdr(options):
    verbose("Reading FLASH header data from EEPROM...")
    vpd = get_vpd_ef(options)
    raw = _debug
    err, lines = vpd.dump_header(raw=raw)
    if err:
        error("Error reading FLASH header from EEPROM:", err)
    else:
        verbose('FLASH header dump:')
        if raw:
            err, temp = vpd.check_data()
            if err:
                verbose('NOTE: HEADER IS NOT VALID! Debug mode, forcing dump.')
                verbose('%s' % err)
                err = None
        if isinstance(lines, list):
            for line in lines:
                print line # Non-verbose
    r = err and 1 or 0
    return r

FuncLine = lambda: inspect.stack()[1][2]
FuncName = lambda: inspect.stack()[1][3]
CallerLine = lambda: inspect.stack()[2][2]
CallerName = lambda: inspect.stack()[2][3]

def dump_traceback(msg=None):
    msg and error(msg)
    traceback.print_exc()

def unhandled_exception():
    etype, evalue = sys.exc_info()[:2]
    dump_traceback("Caught Unhandled Exception: %s" % evalue)

def divceiling(val1, val2):
    return val1 / val2 + (val1 % val2 and 1 or 0)

def position(data, element):
    return [i for i, d in enumerate(data) if d == element]

def bytesize(size, prec=2, stripzeros=False, showB=True, space=True):
    suffixes, index = 'KMGTPEZY', -1
    while size > 1024:
        index += 1 ; size /= 1024.
    if index < 0:
        suffix = ''
    elif index < len(suffixes):
        suffix = suffixes[index]
    else:
        suffix = '?'
    num = "%.*f" % (prec, size)
    if stripzeros:
        num = num.rstrip('0')
    suffix = showB and ("%sB" % suffix) or suffix or 'B'
    return "%s%s%s" % (num, space and ' ' or '', suffix)

def system_command_output(args, shell=True, ignore_stderr=False):
    pipe = subprocess.PIPE
    # Due to annoyances with '*' wildcards, and shell=True or False, etc,
    # always pass a string to Popen.
    if isinstance(args, basestring):
        cmd = args
    else:
        cmd = ' '.join(args)
    p = subprocess.Popen(cmd, stdout=pipe, stderr=pipe,
                         close_fds=True, shell=shell)
    o, e = p.communicate()
    rc = p.returncode or (not ignore_stderr and e) and 1 or 0
    return (rc, filter(None, e.split('\n')), filter(None, o.split('\n')))

def get_tempfile():
    tmpfd, tmpfile = tempfile.mkstemp()
    valid = False
    retry_count = 0
    max_retry = 20
    if tmpfile:
        while not valid and (retry_count < max_retry):
            filename = tmpfile.split('/')[-1]
            valid = re.match('^[a-zA-Z0-9_]*$', filename) is not None
            if ('_z' or '_Z') in filename or not valid:
                valid = False
                delete_file(tmpfile, tmpfd)
                tmpfd, tmpfile = tempfile.mkstemp()
                retry_count += 1

    return tmpfd, tmpfile

def delete_file(filename, fd=None):
    rc = 0
    try:
        if os.path.exists(filename) and os.path.isfile(filename):
            cmd = ['rm', '-f', filename]
            rcc, err, out = system_command_output(cmd)
            if fd is not None:
                os.close(fd)
            # Check rcc, err
            if rcc or err:
                print cmd
                print ((rcc, err, out))
    except:
        unhandled_exception()
        error("Error deleting %s" % filename)
        rc = 1
    return rc

def move_file(source, dest):
    rc = 0
    fdbg("FS - move_file source=%s dest=%s" % (source, dest))
    if not os.path.exists(source) or not os.path.isfile(source):
        error("%s is not a valid file: Move aborted" % source)
        return 1
    try:
        ddir = os.path.dirname(dest)
        if not os.path.exists(ddir):
            os.makedirs(ddir)
        if not os.path.exists(ddir):
            error("Unable to create dir for %s: Move aborted" % dest)
            return 1
        cmd = ['mv', '-f', source, dest]
        fdbg("FS - move_file cmd=%s" % cmd)
        rcc, err, out = system_command_output(cmd)
        fdbg("FS - move_file rcc=%s" % rcc)
        fdbg("FS - move_file err=%s" % err)
        fdbg("FS - move_file out=%s" % out)
        # Check rcc, err
        if rcc or err:
            print cmd
            print ((rcc, err, out))
    except:
        unhandled_exception()
        error("Error moving %s to %s" % (source, dest))
        rc = 1
    fdbg("FS - move_file exit rc=%s" % rc)
    return rc

def copy_file(source, dest):
    rc = 0
    if not os.path.exists(source) or not os.path.isfile(source):
        error("%s is not a valid file: Copy aborted" % source)
        return 1
    try:
        ddir = os.path.dirname(dest)
        if not os.path.exists(ddir):
            os.makedirs(ddir)
        if not os.path.exists(ddir):
            error("Unable to create dir for %s: Copy aborted" % dest)
            return 1
        cmd = ['cp', '-f', source, dest]
        rcc, err, out = system_command_output(cmd)
        # Check rcc, err
        if rcc or err:
            print cmd
            print ((rcc, err, out))
    except:
        unhandled_exception()
        error("Error copying %s to %s" % (source, dest))
        rc = 1
    return rc


class FlashFileSystemError(Exception):
    pass

class FlashFileSystem(object):
    _default_meta_file = files.node_flash_meta_data_file()
    _default_data_file = files.node_flash_data_file()
    _default_mfg_dev = MFG_PARTITION

    _sector_sep = ':'
    _line_sep = ','

    def sectors_format(sectors):
        _sector_sep = ',' # For display, use CSV, not ':'
        result = None
        if sectors is None:
            result = "None"
        else:
            sectors = sorted(sectors)
            consecutive = reduce(
                lambda a, b: a and b,
                map(lambda x: x == 1,
                    map(lambda (a, b): b - a,
                        zip(sectors[:-1], sectors[1:]))), True)
            if consecutive and len(sectors) > 1:
                result = "%s-%s" % (sectors[0], sectors[-1])
            else:
                result = _sector_sep.join(map(str, sectors))
        return result

    def sectors_convert_in(sectors):
        fdbg("FS - sectors_convert_in - sectors=%s" % sectors)
        _sector_sep = ':'
        try:
            result = map(int, sectors.split(_sector_sep))
            fdbg("FS - sectors_convert_in converted sectors=%s" % result)
        except:
            error("Unable to convert sectors: %s" % sectors)
            result = None
        return result

    def sectors_convert_out(sectors):
        fdbg("FS - sectors_convert_out sectors=%s" % sectors)
        _sector_sep = ':'
        try:
            result = _sector_sep.join(map(str, sectors))
            fdbg("FS - sectors_convert_out converted sectors=%s" % result)
        except:
            error("Unable to convert sectors: %s" % sectors)
            result = None
        return result

    def timestamp_format(timestamp):
        return time.ctime(timestamp).rstrip()

    _line_components = [
        'filename',
        'sector',
        'filesize',
        'compsize',
        'timestamp',
        ]
    _line_formats = {
        'filename': {'align':-1, 'format':str, 'in':str, 'out':str},
        'sector': {'align':1, 'format':sectors_format,
                    'in':sectors_convert_in, 'out':sectors_convert_out},
        'filesize': {'align':1, 'format':str, 'in':int, 'out':int},
        'compsize': {'align':1, 'format':str, 'in':int, 'out':int},
        'timestamp': {'align':-1, 'format':timestamp_format,
                      'in':int, 'out':int},
        }

    # filename1,timestamp,filesize,compsize,s:e:c:t:o:r:s
    # filename2,timestamp,filesize,compsize,s:e:c:t:o:r:s
    # s:e:c:t:o:r:s
    # Last line is for sectors containing file system meta file backup
    # These meta sectors are at the end of the LAST write sectors,
    # and determine the START for the NEXT write sectors (round robin)

    def __init__(self, options={}):
        fdbg("FS - __init__ options=%s" % options)
        self._options = options
        fdbg("FS - __init__ _options=%s" % self._options)
        self._compression = options.get('compression', True)
        self._flash_size = self._flash_sector_size = None
        self._mfg_dev = self._default_mfg_dev
        fdbg("FS - __init__ set _mfg_dev=%s" % self._mfg_dev)
        fdbg("FS - __init__ set flash size=%s" % self._flash_size)
        self._meta_file = self._default_meta_file
        fdbg("FS - __init__ set meta_file=%s" % self._meta_file)
        self._data_file = self._default_data_file
        fdbg("FS - __init__ set data_file=", self._data_file)
        self._file_info, self._meta_info, self._meta_data_loaded = {}, {}, {}
        fdbg("FS - __init__ before load")
        self.load()
        fdbg("FS - __init__ after load")
        if not self.sector_count():
            fdbg("FS - __init__  FlashFileSystemError")
            raise FlashFileSystemError("Invalid Flash, unable to init")
        fdbg("FS - __init__ ret 0")
        return 0

    def implementation_error(self):
        fdbg("FS - implementation_error  FlashFileSystemError")
        raise FlashFileSystemError(
            "Function %s must be implemented in subclass" % CallerName())

    def options(self, key=None, default=None):
        fdbg("FS - options start _options=%s" % self._options)
        if key is None:
            fdbg("FS - options key is None")
            return self._options
        else:
            fdbg("FS - options get key=%s" % key)
            opt = self._options.get(key, default)
            fdbg("FS - options get key=%s value=%s" % (key, opt))
            return opt

    def meta_info(self, key=None, default=None):
        if key is None:
            return self._meta_info
        else:
            return self._meta_info.get(key, default)

    def meta_info_file(self, filename=None, add=False):
        fdbg("FS - meta_info_file meta_info=%s" % self._meta_info)
        if filename is None:
            fdbg("FS - meta_info_file filename none return self")
            return self._meta_info
        if not filename in self._meta_info and add:
            self._meta_info[filename] = {}
        fdbg("FS - meta_info_file filename=%s" % filename)
        fdbg("FS - meta_info_file info=%s" % self._meta_info.get(filename))
        return self._meta_info.get(filename)

    def file_info(self, filename=None, add=False):
        if filename is None:
            fdbg("FS - file_info filename none, info=%s" % self._file_info)
            return self._file_info
        if not filename in self._file_info and add:
            self._file_info = {}
            self._file_info[filename] = {'filename': filename}
        fdbg("FS - file_info filename=%s" % filename)
        fdbg("FS - file_info info=%s" % self._file_info.get(filename))
        return self._file_info.get(filename)

    def flash_size(self):
        if self._flash_size is None:
            self._flash_size = self.get_flash_size()
        fdbg("FS - _flash_size=%s" % self._flash_size)
        return self._flash_size

    def sector_size(self):
        if self._flash_sector_size is None:
            self._flash_sector_size = self.get_flash_sector_size()
        fdbg("FS - _flash_sector_size=%s" % self._flash_sector_size)
        return self._flash_sector_size

    def set_next_sector(self, cur_sector, cur_size):
        fsize = self.sector_size()
        numsec = cur_size / self.sector_size()
        if cur_size % self.sector_size():
            numsec = numsec + 1
        fdbg("FS - set_next_sector - cur_sector=%s" % cur_sector)
        fdbg("FS - set_next_sector - cur_size=%s" % cur_size)
        nextsecint = cur_sector + numsec
        nextsec = []
        nextsec.append(nextsecint)
        fdbg("FS - set_next_sector - nextsecint=%s" % nextsecint)
        fdbg("FS - set_next_sector - nextsec=%s" % nextsec)
        return nextsec

    def sector_count(self):
        fdbg("FS - sector_count - start")
        flash_size = self.flash_size()
        sector_size = self.sector_size()
        fdbg("FS - sector_count - flash_size=%s" % flash_size)
        fdbg("FS - sector_count - sector_size=%s" % sector_size)
        if not flash_size or not sector_size:
            fdbg("FS - sector_count - bad sizes")
            error("Invalid flash: flash size %s, sector size %s" %
                  (flash_size, sector_size))
            return None
        if flash_size % sector_size:
            fdbg("FS - sector_count - sizes obtained")
            bytef = lambda s: bytesize(s, prec=8, stripzeros=True)
            fsize = bytef(flash_size)
            ssize = bytef(sector_size)
            count = ("%.8f" % (float(flash_size) / sector_size)).rstrip('0')
            error("Invalid flash: flash size %s, sector size %s, "
                  "sector count = %s != int" % (fsize, ssize, count))
            return None
        fdbg("FS - sector_count - ret=%s" % (flash_size / sector_size))
        return flash_size / sector_size

    def required_sector_count(self, size):
        sector_size = self.sector_size()
        fdbg("FS - required_sector_count - sector_size=%s" % sector_size)
        fdbg("FS - required_sector_count - size=%s" % size)
        if not sector_size:
            error("Invalid flash: sector size %s" % sector_size)
            return None
        reqsec = divceiling(size, sector_size)
        fdbg("FS - required_sector_count - reqsec=%s" % reqsec)
        return reqsec

    #parse through the _meta_data_loaded files for used sectors
    def used_meta_data(self):
        seclist = []
        fmts = self._line_formats
        comps = self._line_components
        fdbg("FS - used_meta_data - _meta_data_loaded=%s" %
                self._meta_data_loaded)
        # parse for sectors in _meta_data_loaded
        for fline in self._meta_data_loaded.values():
            fdbg("FS - used_meta_data get ckline - fline=%s" % fline)
            sec = fline.get('sector')
            fdbg("FS - used_meta_data sec=%s" % sec)
            for s in sec:
                fdbg("FS - used_meta_data s=%s" % s)
                seclist.append(s)
        fdbg("FS - used_meta_data seclist=%s" % seclist)
        usedlist = []
        if seclist:
            usedlist = sorted(seclist)
        fdbg("FS - used_meta_data sorted seclist=%s" % usedlist)
        return usedlist

    def used_free_sectors(self):
        # 1. free up previous meta sectors, put in free
        # 2. using previous round robin sector, set meta to next avail sectors
        #    and remove from free
        sectors = range(self.sector_count())
        usage = self.usage()
        used = usage and list(zip(*usage)[0]) or []
        fdbg("used_free_sectors - meta info=%s" % self._meta_info)
        meta = self.used_meta_data()
        if meta is None:
            meta = []
        fdbg("used_free_sectors - meta (sectors)=%s" % meta)
        #used = list(set(used).difference(meta))
        used = meta
        free = list(set(sectors).difference(used))
        fdbg("used_free_sectors - used=%s" % used)
        fdbg("used_free_sectors - free=%s" % free)
        # Get the next round robin starting sector, based on meta (last writes)
        # Return free list starting with this next available sector
        if meta:
            last = free[0]
        else:
            last = 0 # none use first
        fdbg("used_free_sectors - last=%s" % last)
        findpos = position(free, last)
        pos = findpos and findpos[0] or 0
        fdbg("used_free_sectors - pos=%s findpos=%s" % (pos, findpos))
        free = free[pos:] + free[:pos]
        fdbg("used_free_sectors - free=%s used=%s" % (free, used))
        return used, free

    def available_sectors(self, size):
        count = self.required_sector_count(size)
        fdbg("available_sectors - count=%s" % count)
        if count is None:
            return None
        sectors = None
        # IMPORTANT: free needs to be large enough for the file sectors
        reqd_count = count
        used, free = self.used_free_sectors()
        if not free:
            error("No free sectors available (flash is full)")
        elif len(free) < reqd_count:
            error("Insufficient sectors: required %d, available %s" %
                  (reqd_count, len(free)))
        else:
            fdbg("available_sectors - reqd_count=%s" % reqd_count)
            sectors = free[:reqd_count]
            fdbg("available_sectors - sectors=%s" % sectors)
        return sectors

    def numeric(self, value):
        return isinstance(value, int) or isinstance(value, long)

    def positive(self, value):
        return self.numeric(value) and value > 0

    def nonnegative(self, value):
        return self.numeric(value) and value >= 0

    def valid_file_info(self, filename, write=False, skipmsg=False):
        # Returns file info for filename if valid, else None
        # Reports errors found
        try:
            if write:
                if not os.path.exists(filename):
                    error("File %s does not exist" % filename)
                    return None
                if not os.path.isfile(filename):
                    error("File %s is not a valid file" % filename)
                    return None
            fdbg("FS - valid_file_info filename=%s" % filename)
            info = self.file_info(filename)
            if write:
                if info:
                    msg = "Overwriting %s in flash" % filename
                else:
                    msg = "Adding %s to flash" % filename
                    info = self.file_info(filename, add=True)
                fstat = os.stat(filename)
                info['filesize'] = fstat.st_size
                info['timestamp'] = fstat.st_mtime
                info['compsize'] = None # Set in write()
                info['sector'] = None # Set in write()
                fdbg("FS - valid_file_info info=%s" % info)
            else:
                msg = None
                if not info:
                    error("%s not found in flash file system" % filename)
                    return None
            if not filename == info.get('filename'):
                error("%s has inconsistent metadata: filename=%s" %
                      (filename, info.get('filename')))
                return None
            filesize = info.get('filesize')
            if not self.nonnegative(filesize):
                error("%s has invalid metadata: filesize=%s" %
                      (filename, filesize))
                return None
            sectors = info.get('sector')
            valid = sectors and reduce(lambda a, b: a and b,
                                       map(self.nonnegative, sectors))
            if (not sectors or not valid) and not write:
                error("%s has invalid metadata: sectors=%s" %
                      (filename, sectors))
                return None
            if not skipmsg:
                if msg:
                    verbose(msg)
        except:
            unhandled_exception()
            info = None
        fdbg("FS - valid_file_info info=%s" % info)
        return info

    def compress(self, filename):
        compsize = 0
        outfile = tmpfd = tmpfile = None
        info = self.file_info(filename)
        fdbg("FS - compress after file_info info=%s" % info)
        fdbg("FS - compress after filename=%s" % filename)
        if not info:
            error("%s not found in flash file system" % filename)
            return None
        filesize = info.get('filesize')
        if None in [filesize]:
            error("%s has invalid flash file metadata" % filename)
            return None
        try:
            if self._compression:
                fdbg("FS - compress option enabled")
                tmpfd, tmpfile = get_tempfile()
                if tmpfile:
                    cmd = ['gzip', '-cfS', '""', filename, ">", tmpfile]
                    fdbg("FS - compress cmd=%s" % cmd)
                    rcc, err, out = system_command_output(cmd)
                    fdbg("FS - compress rcc=%s" % rcc)
                    fdbg("FS - compress err=%s" % err)
                    fdbg("FS - compress out=%s" % out)
                    # Check rcc, err
                    if rcc or err:
                        print cmd
                        print ((rcc, err, out))
                    if _fdbg:
                        cmd = ['cp', '-f', tmpfile, '/tmp/flashzipout']
                        print "FS - compress cmd=", cmd
                        rcc, err, out = system_command_output(cmd)
                        print "FS - compress rcc=", rcc
                        print "FS - compress err=", err
                        print "FS - compress out=", out
                    fstat = os.stat(tmpfile)
                    compsize, outfile = fstat.st_size, tmpfile
                    fdbg("FS - compress passed compsize=%s" % compsize)
                    fdbg("FS - compress passed outfile=%s" % outfile)
                else:
                    # Continue with uncompressed save
                    compsize, outfile = 0, filename
            else:
                fdbg("FS - compress option disabled")
                # Continue with uncompressed save
                compsize, outfile = 0, filename
        except:
            unhandled_exception()
            compsize, outfile = -1, None
        finally:
            if compsize < 0:
                compsize, outfile = 0, None
                fdbg("FS - compress failed compsize=%s" % compsize)
                fdbg("FS - compress failed outfile=%s" % outfile)
                error("Failed to compress %s" % filename)
                if tmpfile:
                    # Don't set result to fail for this ??
                    delete_file(tmpfile, tmpfd)
        info['compsize'] = compsize
        fdbg("FS - compress exit compsize=%s outfile=%s" %
                (compsize, outfile))
        return outfile

    def decompress(self, info, filename, tmpfile, outfile, retry):

        fdbg("FS - decompress INIT*********************")
        fdbg("FS - decompress info=%s filename=%s" %
                (info, filename))
        if not info:
            error("%s not found in flash file system" % filename)
            return 1
        filesize = info.get('filesize')
        compsize = info.get('compsize')
        if None in [filesize, compsize]:
            error("%s has invalid flash file metadata" % filename)
            return 1
        if not tmpfile:
            error("%s temp file not available for flash data" % filename)
            return 1
        if not os.path.exists(tmpfile) or not os.path.isfile(tmpfile):
            error("%s temp file %s does not exist" % (filename, tmpfile))
            return 1
        rc = 0
        if compsize:
            verbose("Uncompressing %s: compsize=%s" %
                  (filename, compsize))
            cmd = ['gunzip', '-dfS', '""', tmpfile]
            fdbg("FS - decompress cmd=%s" % cmd)
            rcc, err, out = system_command_output(cmd)
            fdbg("FS - decompress rcc=%s" % rcc)
            fdbg("FS - decompress err=%s" % err)
            fdbg("FS - decompress out=%s" % out)
            # Check rcc, err
            if rcc or err:
                rc = 1
                if retry >= _max_attempts:
                    print cmd
                    print ((rcc, err, out))
            # Move to final destination
            # check decompression for debug
            if _fdbg:
                cmd = ['ls', '-al', tmpfile]
                print "FS - decompress cmd=", cmd
                rcc, err, out = system_command_output(cmd)
                print "FS - decompress rcc=", rcc
                print "FS - decompress err=", err
                print "FS - decompress out=", out
            # save decompression file for debug
            if _fdbg:
                cmd = ['cp', '-f', tmpfile, '/tmp/flashzipin']
                print "FS - decompress cmd=", cmd
                rcc, err, out = system_command_output(cmd)
                print "FS - decompress rcc=", rcc
                print "FS - decompress err=", err
                print "FS - decompress out=", out
        if outfile and not rcc:
            fdbg("FS - decompress move %s to %s" % (tmpfile, outfile))
            if move_file(tmpfile, outfile):
                rc = 1
        elif not outfile:
            error("No output file specified for flash data")
            # Do not return an error for this case?
        fdbg("FS - decompress exit rc=%s" % rc)
        return rc

    def _read(self, info, outfile):
        self.implementation_error()

    # read filename - file location in OneFS (tmpfile)
    # outfile - desired destination location in OneFS
    # altfile - if present, final desired destination location in OneFS
    #     when restoring CTO files to /tmp instead of MFG_DIR
    def read(self, filename, outfile=None, altfile=None):
        result = 1
        trynumber = 0
        tmpfd = tmpfile = None
        fdbg("FS - read filename=%s" % filename)
        finalfilename = filename
        if outfile:
            finalfilename = outfile
            fdbg("FS - read outfile=%s" % outfile)
        if altfile:
            finalfilename = altfile
            fdbg("FS - read altfile=%s" % altfile)
        # check OneFS filename/path exists, create if dir missing
        if not os.path.exists(finalfilename):
            # file doesn't exist
            fdbg("FS - read finalfilename=%s does not exist" %
                    finalfilename)
            filedir = os.path.dirname(finalfilename)
            if not os.path.exists(filedir):
                #directory doesn't exist either
                fdbg("FS - read finalfilename=%s path %s does not exist" %
                        (finalfilename, filedir))
                self.mkdirsfrompath(filedir)
            else:
                fdbg("FS - read file dir=%s exists" % filedir)
        else:
            fdbg("FS - read finalfilename=%s exists" % finalfilename)
        info = self.search_file_info(filename)
        fdbg("FS - read after search_file_info=%s filename=%s" %
                (info, filename))
        if not info:
            print("=> FAILED read %s: File does not exist" % finalfilename)
            return result
        try:
            # If compressed, need to read to a tempfile.
            # Always read to a tempfile, then copy to outfile after.
            tmpfd, tmpfile = get_tempfile()

            if tmpfile:
                fdbg("FS - read data into %s" % tmpfile)
                fdbg("FS - read move final data into %s" % finalfilename)
                result = 0 # zero for passed
                if _fdbg:
                    cmd = ['ls', '-al', tmpfile]
                    print "FS - read cmd=", cmd
                    rcc, err, out = system_command_output(cmd)
                    print "FS - read rcc=", rcc
                    print "FS - read err=", err
                    print "FS - read out=", out
                while trynumber < _max_attempts:
                    trynumber = trynumber + 1
                    bytesread = self._read(info, tmpfile)
                    if _fdbg:
                        cmd = ['ls', '-al', tmpfile]
                        print "FS - read cmd=", cmd
                        rcc, err, out = system_command_output(cmd)
                        print "FS - read rcc=", rcc
                        print "FS - read err=", err
                        print "FS - read out=", out
                        cmd = ['cat', tmpfile]
                        print "FS - read cmd=", cmd
                        rcc, err, out = system_command_output(cmd)
                        print "FS - read rcc=", rcc
                        print "FS - read err=", err
                        cmd = ['cp', '-f', tmpfile, '/tmp/flashread.out']
                        print "FS - read cmd=", cmd
                        rcc, err, out = system_command_output(cmd)
                        print "FS - read rcc=", rcc
                        print "FS - read err=", err
                        print "FS - read out=", out
                    fdbg("FS - read %s bytes of data into %s" %
                         (bytesread, tmpfile))
                    if bytesread is None or bytesread <= 0:
                        result = 1
                    elif result == 0:
                        result = self.decompress(info, filename, tmpfile,
                                                 finalfilename, trynumber)
                        # We call decompress to move tmpfile to finalfilename,
                        # even if it is not actually compressed
                    if result == 0:
                        break
            else:
                error("No temp file available for flash data")
                result = 1
        except:
            unhandled_exception()
            error("Failed to read %s from flash" % filename)
            result = 1
        finally:
            fdbg("FS - read finally result=%s" % result)
            if tmpfile:
                delete_file(tmpfile, tmpfd)
            if result is 0 and bytesread: # Success
                fdbg("FS - read %s: result=%s" %
                      (finalfilename, result))
                print("=> PASSED read %s" % finalfilename)
            else: # Failure
                fdbg("FS - read %s: result=%s" %
                      (finalfilename, result))
                print("=> FAILED read %s" % finalfilename)
        fdbg("FS - read exit result=%s" % result)
        return result

    def _write(self, info):
        self.implementation_error()

    def write(self, filename):
        result = 1
        outfile = sectors = None
        saveinfo = info = self.valid_file_info(filename, write=True)
        fdbg("FS - write after valid_file_info=%s filename=%s" %
                (info, filename))
        if not info:
            return result
        try:
            # delete meta data if file to write is already in FLASH meta data
            self._meta_info = {}
            self._meta_info[0] = info
            fdbg("FS - write before check_dup_metadata info=%s" % info)
            fdbg("FS - write before check_dup_metadata self._meta_info=%s" %
                    self._meta_info)
            if self.check_dup_metadata():
                # duplicate removed from self._meta_data_loaded, update file
                fdbg("FS - write ck save metadata after check_dup_metadata")
                # remove existing file from flash meta data and save
                # if not empty
                self._meta_info = {}
                if len(self._meta_data_loaded):
                    fdbg("FS - write save metadata after check_dup_metadata")
                    if self.save_metadata():
                        error("Failed to delete existing %s from metadata" %
                              (filename))
                        return result
                    if self.file_system_load():
                        error("Failed to reload metadata after deleting %s" %
                              (filename))
                        return result
                    saveinfo = info = self.valid_file_info(filename, write=True)
                # set info from saved _file_info again
                info = saveinfo
                fdbg("FS - write after save_metadata info=%s" % info)
                fdbg("FS - write after save_metadata _meta_info=%s" %
                        self._meta_info)
                fdbg("FS - write after save_metadata _meta_data_loaded=%s" %
                        self._meta_data_loaded)
            ###
            # We call compress to move filename to a tempfile,
            # even if it is not actually compressed
            outfile = self.compress(filename)
            if not outfile:
                error("Failed to write %s to flash: compression failed" %
                      (filename))
                return result
            filesize = info.get('filesize')
            compsize = info.get('compsize')
            fsize = compsize or filesize
            fdbg("FS - write before available_sectors info=%s" % info)
            fdbg("FS - write before available_sectors _meta_info=%s" %
                    self._meta_info)
            fdbg("FS - write before available_sectors _meta_data_loaded=%s" %
                    self._meta_data_loaded)
            sectors = self.available_sectors(fsize)
            fdbg("FS - write available_sectors=%s" % sectors)
            if not sectors:
                fsize = "file size %s" % bytesize(filesize)
                csize = compsize and " (compressed %s)" % bytesize(compsize)
                error("Failed to write %s to flash: Insufficient sectors for "
                      "%s%s" % (filename, fsize, csize))
                return result
            # set sector to be a list of int sector numbers
            info['sector'] = sectors
            self._flash_first_free_sector = sectors[0]
            fdbg("FS - write self._flash_first_free_sector=%s" %
                    self._flash_first_free_sector)
            fdbg("FS - write info=%s outfile=%s" % (info, outfile))
            result = 0 # set for passed
            # None on error, else bytecount
            byteswritten = self._write(info, outfile)
            fdbg("FS - write byteswritten=%s" % byteswritten)
            if byteswritten is None or byteswritten <= 0:
                result = 1
        except:
            unhandled_exception()
            error("Failed to write %s to flash" % filename)
            result = 1
        finally:
            fdbg("FS - write finally result=%s" % result)
            if outfile and outfile != filename:
                # Don't set result to fail for this ??
                delete_file(outfile)
            fdbg("FS - write info=%s" % info)
            self._meta_info = {}
            self._meta_info[0] = info
            fdbg("FS - write do save for file written metadata")
            fdbg("FS - write do save _meta_info=%s" % self._meta_info)
            fdbg("FS - write do save _meta_data_loaded=%s" %
                    self._meta_data_loaded)
            if not result:
                if self.save():
                    result = 1
            if result: # Failure
                fdbg("FS - write %s: result=%s" % (filename, result))
                print("=> FAILED write %s" % filename)
            else: # Success
                fdbg("FS - write %s: result=%s sectors=%s" %
                      (filename, result, sectors))
                print("=> PASSED write %s" % filename)
            fdbg("FS - write end _meta_info=%s" % self._meta_info)
            fdbg("FS - write end _meta_data_loaded=%s" % self._meta_data_loaded)
        fdbg("FS - write exit result=%s" % result)
        return result

    def _delete(self, info, outfile):
        self.implementation_error()

    def delete(self, filename):
        result = 0
        fdbg("FS - delete filename=%s" % filename)
        tmpfd = tmpfile = None
        info = {}
        info['filename'] = filename
        fdbg("FS - delete info=%s" % info)
        if not info:
            return None
        try:
            self._meta_info[0] = info
            fdbg("FS - delete - _meta_info=%s" % self._meta_info)
            fdbg("FS - delete - _meta_data_loaded=%s" % self._meta_data_loaded)
            if not self.check_dup_metadata():
                error("Unable to delete %s from flash" % filename)
                result = 1
        except:
            unhandled_exception()
            error("Failed to delete %s from flash" % filename)
            result = 1
        self._meta_info = {}
        if self.save_metadata():
            error("Failed to save delete flash metadata for %s" % filename)
            result = 1
        return result

    # If not already mounted mount MFG_PARTITION as MFG_DIR
    # returns initial mount state to caller 1=mounted
    def mountmfg(self):
        result = 0
        alreadymnted = 1
        ddev = self._mfg_dev
        ddir = MFG_DIR
        fdbg("FS - mountmfg ddev=%s" % ddev)
        try:
            if not fs_mounted(MFG_PARTITION):
                if not os.path.exists(ddir):
                    os.makedirs(ddir)
                if not os.path.exists(ddir):
                    error('Unable to create dir "%s" for mounting "%s"' %
                            ddir, ddev)
                    result = 1
                    return result, alreadymnted
                cmd = ['mount', ddev, ddir]
                fdbg("FS - mountmfg cmd=%s" % cmd)
                rcc, err, out = system_command_output(cmd)
                fdbg("FS - mountmfg rcc=%s" % rcc)
                fdbg("FS - mountmfg err=%s" % err)
                fdbg("FS - mountmfg out=%s" % out)
                # Check rcc, err
                if rcc or err:
                    print cmd
                    print ((rcc, err, out))
                    result = 1
                alreadymnted = 0
        except:
            unhandled_exception()
            error("Error mount %s as %s" % (ddev, ddir))
            result = 1
        fdbg("FS - mountmfg exit result=%s alreadymnted=%s" %
                (result, alreadymnted))
        return result, alreadymnted

    # /dev/mirror/mfg was not already mounted as /mfg
    # this routine will umount /mfg
    def umountmfg(self):
        result = 0
        ddir = MFG_DIR
        msg = unmount_fs(ddir)
        if msg:
            error("Error umount %s: %s" % (ddir, msg))
            result = 1
        fdbg("FS - umountmfg exit result=%s" % result)
        return result

    # for each file in /etc/isilon_flash_files copy (write) it and its checksum to Flash
    def copycto(self):
        result = 0
        dfile = self._data_file
        if not os.path.exists(dfile):
            print "%s does not exist, unable to copy files" % dfile
            result = 1
            return result
        rc, alreadymounted = self.mountmfg()
        if not rc:
            with open(dfile) as f:
                data = f.readlines()
                fdbg("FS - copycto read data=%s" % data)
            for filename in data:
                filename = filename.rstrip('\r\n')
                if not os.path.exists(filename):
                    print "CTO %s does not exist, skipped" % filename
                    continue
                fdbg("FS - generate md5 file for %s" % filename)
                md5filename = "%s.md5" % filename
                md5file = os.open(md5filename, os.O_WRONLY | os.O_CREAT, 0777)
                rcc, err, checksum = system_command_output("md5 -q %s" % filename)
                if rcc or err:
                    result = 1
                    print "unable to create md5 file for %s" % filename
                    continue
                os.write(md5file,"%s\n" % str(checksum[0]))
                os.close(md5file)
                fdbg("FS - copycto parse readlines filename=%s" %
                        filename)
                wrc = self.write(filename)
                if not wrc:
                    print "CTO %s copied to Flash" % filename
                    # reload meta data from new file copied for next one
                    rc = self.file_system_load()
                    if rc == 1:
                        print "CTO %s failed to update meta data" % filename
                        result = 1
                else:
                    result = 1
                    print "CTO %s failed to copy to Flash" % filename
                #write checksum to flash
                wrm = self.write(md5filename)
                if not wrm:
                    print "CTO checksum %s copied to Flash" % md5filename
                    rc = self.file_system_load()
                    if rc == 1:
                        print "CTO %s failed to update meta data" % md5filename
                        result = 1
                else:
                    result = 1
                    print "CTO checksum %s failed to copy to Flash" % md5filename
                os.remove(md5filename)
            if not alreadymounted:
                self.umountmfg()
        fdbg("FS - copycto exit result=%s" % result)
        return result

    # for each file in /etc/isilon_flash_files restore (copy) it from Flash
    # restore files to dir as specified in /etc/isilon_flash_files if
    # restoremfg is 1
    # restore files to /tmp if restoremfg is 0
    def restorecto(self, restoremfg, md5=False):
        result = 0
        dfile = self._data_file
        if not os.path.exists(dfile):
            print "%s does not exist, unable to restore files" % dfile
            result = 1
            return result
        if restoremfg:
            altdir = None
        else:
            altdir = '/tmp'
        fdbg("restorecto - altdir=%s" % altdir)
        rc, alreadymounted = self.mountmfg()
        if not rc:
            with open(dfile) as f:
                data = f.readlines()
                fdbg("FS - restorecto read data=%s" % data)
            for filename in data:
                filename = filename.rstrip('\r\n')
                if md5:
                    filename = "%s.md5" % filename
                fdbg("FS - restorecto parse readlines filename=%s" %
                        filename)
                if restoremfg:
                    # restore file to location per /etc/isilon_flash_files
                    rrc = self.read(filename, filename)
                else:
                    # restore file to /tmp/xyz via altfile
                    fnameonly = os.path.basename(filename)
                    fdbg("restorecto - base filename=%s to /tmp" % fnameonly)
                    altfile = "/tmp/" + fnameonly
                    fdbg("restorecto - %s to altdir=%s" % (filename, altfile))
                    rrc = self.read(filename, filename, altfile)
                if not rrc:
                    if altdir:
                        print("CTO %s restored from Flash to %s" %
                                (filename, altfile))
                        if not md5:
                            cmd = 'diff %s %s' % (filename,altfile)
                        else:
                            cmd = 'md5 -q %s | diff %s -' % (filename.rstrip('.md5'), altfile)
                        fdbg("FS - move_file cmd=%s" % cmd)
                        rcc, err, out = system_command_output(cmd)
                        fdbg("FS - move_file rcc=%s" % rcc)
                        fdbg("FS - move_file err=%s" % err)
                        fdbg("FS - move_file out=%s" % out)
                        # Check rcc, err
                        if rcc or err:
                            print("CTO %s diff bad with %s" %
                                    (filename, altfile))
                        else:
                            print("CTO %s diff ok with %s" %
                                    (filename, altfile))
                    else:
                        print "CTO %s restored from Flash" % filename
                else:
                    result = 1
                    print "CTO %s failed to restore from Flash" % filename
            if not alreadymounted:
                self.umountmfg()
        fdbg("FS - restorecto exit result=%s" % result)
        return result

    def _load(self):
        # If anything specific needs to be done in __init__,
        # define in child class - e.g., FakeFlash loads from data file
        pass

    def load(self):
        rc = 0
        vpdopt = self.options('vpd')
        initopt = vpdopt.get('init')
        fdbg("FS - load - start initopt=%s" % initopt)
        fdbg("FS - load - start vpdopt=%s" % vpdopt)
        if initopt:
            fdbg("FS - load - call file_system_init")
            rc = self.file_system_init()
        if rc == 0:
            fdbg("FS - load - call file_system_load")
            rc = self.file_system_load()
            if rc == 0:
                fdbg("FS - load - call _load")
                rc = self._load()
        return rc

    def _save(self):
        # If anything specific needs to be done before saving metadata,
        # define in child class - e.g, FakeFlash saves to data file
        pass

    def save(self):
        rc = 0
        fdbg("FS - save metadata")
        if self._save():
            rc = 1
        if self.save_metadata():
            rc = 1
        return rc

    def usage(self):
        info = self.file_info()
        keys = ['filename', 'filesize', 'compsize', 'sector']
        udata = [[v.get(k) for k in keys] for v in info.itervalues()]
        udata = filter(lambda data: not None in data, udata)
        ssize = self.sector_size()
        usage = []
        for filename, filesize, compsize, sectors in udata:
            fsize = compsize or filesize
            for sector in sectors:
                size = fsize / ssize and ssize or fsize % ssize
                fsize -= ssize
                usage.append((sector, size, filesize, filename))
        fsize = filesize = self.meta_info('filesize', 0)
        for sector in self.meta_info('sector', []):
            size = fsize / ssize and ssize or fsize % ssize
            fsize -= ssize
            usage.append((sector, size, filesize, 'META DATA FILE'))
        return sorted(usage)

    def display_usage(self, barwidth=40):
        result = ['Flash sector usage:']
        bytef = lambda n: bytesize(n, prec=0, showB=False, space=False)
        ssize = self.sector_size()
        usage = self.usage()
        if not usage:
            return []
        sects, sizes, fsizs, names = zip(*usage)
        sectw = max(map(len, map(str, sects)) + [2])
        sizew = max(map(len, map(str, sizes)))
        fsizw = max(map(len, map(bytef, fsizs)))
        namew = max(map(len, map(str, names)))
        fmt = " ".join(map(
            lambda (a, w, t): "".join(map(str, [a, w, t])),
            [("%", sectw, "d:"), ("%", sizew, "d:"), ("%-", barwidth, "s"),
             ("%", fsizw, "s"), ("%-", namew, "s")]))
        for sect, size, fsize, name in usage:
            width = divceiling(barwidth * size, ssize)
            result.append(fmt % (sect, size, '*' * width, bytef(fsize), name))
        return result

    def check_data(self):
        fdbg("FS - check_data self._meta_file=%s" % self._meta_file)
        dfile = self._data_file
        if not os.path.exists(dfile):
            print dfile, "does not exist - consider finitcto"
            err = 1
        else:
            err = self.file_system_load()
            if err:
                error("check_data flash meta data invalid: Consider re-init?:",
                        err)
            else:
                with open(dfile) as f:
                    data = f.read().splitlines()
                    fdbg("FS - check_data read data=%s" % data)
                for filename in data:
                    info = self.search_file_info(filename)
                    if info == {}:
                        verbose("CTO file %s missing from flash" % filename)
                        err = 1
                    else:
                        fdbg("FS - check_data %s found in flash" % filename)
        if not err:
            verbose("check_data flash meta data appear valid!")
        r = err and 1 or 0
        return r

    # ###########################################################
    # Flash File System "Middle Level" Definitions
    # ###########################################################

    def file_system_init(self):
        rc = 0
        fdbg("FS - file_system_init")
        meta_file = self._meta_file
        sectors = '0' # Start with sector 0 on every init
        self._flash_first_free_sector = int(sectors)
        if os.path.exists(meta_file):
            fdbg("FS - file_system_init - path exits")
            backup_file = "%s.bup" % meta_file
            print("Warning: Destroying existing flash file system %s" %
                  meta_file)
            print("Creating backup file %s..." % backup_file)
            fdbg("FS - file_system_init - copy_file")
            copy_file(meta_file, backup_file) # Ignore errors
        try:
            fdbg("FS - file_system_init - try meta_file=%s" % meta_file)
            meta_file_dir = os.path.dirname(meta_file)
            if not os.path.exists(meta_file_dir):
                print "os.makedirs(%s)" % meta_file_dir
                self.mkdirsfrompath(meta_file_dir)
            open(meta_file, 'w').close
            fstat = os.stat(meta_file)
            self._meta_info = {}
            fdbg("FS - file_system_init - _meta_info=%s" % self._meta_info)
        except:
            unhandled_exception()
            error("Error creating new flash file system %s" % meta_file)
            rc = 1
        if rc is 0:
            self.save()
        return rc

    def _file_data(self, line):
        fdbg("FS - _file_data line=%s" % line)
        line = str(line).rstrip()
        # split string at first ':' char for filename vs rest
        # note multiple sectors will have more ':' chars between sectors!
        kdata = line.split(self._sector_sep, 1)
        # remove quotes around name string
        s1 = kdata[0].replace("'", "")
        # remove any whitespace after filename chars
        filename = s1.rstrip()
        k2 = kdata[1].strip()
        fdbg("FS - _file_data kdata=%s" % kdata)
        fdbg("FS - _file_data filename=%s" % filename)
        fdbg("FS - _file_data k2=%s" % k2)
        data = k2.split(self._line_sep)
        fdbg("FS - _file_data data=%s" % data)
        fdbg("FS - _file_data len(data)=%s" % len(data))
        fdbg("FS - _file_data len(self._line_components)=%s" %
                len(self._line_components))
        if (len(data) + 1) != len(self._line_components):
            print line
            print data
            print self._line_components
            error("Invalid flash file system data: %s" % line)
            return None
        info = {}
        info['filename'] = filename
        try:
            info['filesize'] = self._line_formats['filesize']['in'](data[1])
        except:
            info['filesize'] = None
        fdbg("FS - _file_data filesize=%s" % info.get('filesize'))
        try:
            info['compsize'] = self._line_formats['compsize']['in'](data[2])
        except:
            info['compsize'] = None
        fdbg("FS - _file_data compsize=%s" % info.get('compsize'))
        s2 = data[0].strip()
        fdbg("FS - _file_data data sector s2=%s" % s2)
        info['sector'] = self._line_formats['sector']['in'](s2)
        seclist = info.get('sector')
        cursect = int(seclist[0])
        fdbg("FS - _file_data cursect=%s" % cursect)
        fdbg("FS - _file_data seclist=%s" % seclist)
        if info.get('compsize'):
            cursz = info.get('compsize')
        else:
            cursz = info.get('filesize')
        secs = info.get('sector')
        sectsz = self.sector_size()
        thislastsec = secs[-1]
        fdbg("FS - _file_data thislastsec=%s" % thislastsec)
        fdbg("FS - _file_data _flash_first_free_sector=%s" %
                self._flash_first_free_sector)
        if self._flash_first_free_sector is None:
            fdbg("FS - _file_data 1st sec None set it")
            self._flash_first_free_sector = (
                    self.set_next_sector(thislastsec, sectsz))
        elif self._flash_first_free_sector[0] <= thislastsec:
            fdbg("FS - _file_data 1st sec smaller set it last=%s" %
                  thislastsec)
            fdbg("FS - _file_data  self._flash_first_free_sector=%s" %
                  self._flash_first_free_sector)
            self._flash_first_free_sector = (
                    self.set_next_sector(thislastsec, sectsz))
        else:
            fdbg("FS - _file_data 1st sec bigger, leave it alone")
        fdbg("FS - _file_data _flash_first_free_sector=%s" %
                self._flash_first_free_sector)
        fdbg("FS - _file_data sector=%s" % info.get('sector'))
        try:
            info['timestamp'] = self._line_formats['timestamp']['in'](data[3])
        except:
            info['timestamp'] = None
        fdbg("FS - _file_data timestamp=%s" % info.get('timestamp'))
        fdbg("FS - _file_data info=%s" % info)
        return info

    def _meta_data(self, line):
        fdbg("FS - _meta_data line=%s" % line)
        line = str(line).strip().split(self._sector_sep)
        fdbg("FS - _meta_data strip line=%s" % line)
        try:
            data = map(int, line)
        except:
            data = None
        return data

    def file_system_load(self):
        rc = 0
        self._file_info, self._meta_info, self._meta_data_loaded = {}, {}, {}
        self._flash_first_free_sector = None
        meta_file = self._meta_file
        sectors = None
        fdbg("FS - file_system_load - check os path")
        found = lambda f: os.path.exists(f) and os.path.isfile(f)
        if not found(meta_file):
            # no meta_file, try reading FLASH metadata from EEPROM
            print("Warning: %s missing, attempt restore from EEPROM" %
                    meta_file)
            if self.file_system_pull():
                # failed, reinit
                print("Warning: %s restore from EEPROM attempt failed" %
                        meta_file)
                print("Attempt to create new flash file system...")
                self.file_system_init()
        if not found(meta_file):
            error("No flash file system found")
            return 1
        filesys = []
        fdbg("FS - file_system_load try meta_file=%s" % meta_file)
        try:
            fstat = os.stat(meta_file)
            data = []
            index = 0
            with open(meta_file) as f:
                # read and toss the warning line
                line = f.readline()
                fdbg("FS - file_system_load read line=%s" % line)
                data = f.readlines()
                fdbg("FS - file_system_load readlines data=%s" % data)
            for line in data:
                fdbg("FS - file_system_load parse readlines line=%s" % line)
            fdbg("FS - file_system_load readlines data=%s" % data)
        except:
            unhandled_exception()
            data = []
        self._meta_info = {}
        if not data:
            verbose("Flash file system is empty")
        else:
            fdbg("FS - file_system_load else _file_info=%s" % self._file_info)
            fdbg("FS - file_system_load else _meta_info=%s" % self._meta_info)
            filesys = filter(None, map(self._file_data, data))
            fdbg("FS - file_system_load else filesys=%s" % filesys)
            fdbg("FS - file_system_load sector=%s" %
                    self._meta_info.get('sector'))
        self._file_info = dict()
        index = 0
        for fs in filesys:
            self._file_info[index] = fs
            index = index + 1
        if self._flash_first_free_sector is None:
            self._flash_first_free_sector = 0
        self._meta_data_loaded = self._file_info.copy()
        fdbg("FS - file_system_load else _flash_first_free_sector=%s" %
                self._flash_first_free_sector)
        fdbg("FS - file_system_load nextsector=%s" %
                self._flash_first_free_sector)
        fdbg("FS - file_system_load end _file_info=%s" % self._file_info)
        fdbg("FS - file_system_load end _meta_info=%s" % self._meta_info)
        fdbg("FS - file_system_load end _flash_first_free_sector=%s" %
                self._flash_first_free_sector)
        fdbg("FS - file_system_load end rc=%s" % rc)
        fdbg("FS - file_system_load exit _meta_data_loaded=%s" %
                self._meta_data_loaded)
        return rc

    # search for file name in _meta_data_loaded
    # return None or dictionary of info
    def search_file_info(self, filename):
        result = {}
        fdbg("FS - search_file_info filename=%s" % filename)
        self._meta_info[0] = { 'filename' : filename }
        fmts = self._line_formats
        comps = self._line_components
        fdbg("FS - search_file_info - _meta_info=%s" % self._meta_info)
        fdbg("FS - search_file_info - _meta_data_loaded=%s" %
                self._meta_data_loaded)
        # check for newfn in _meta_data_loaded
        index = 0
        for fline in self._meta_data_loaded.values():
            fdbg("FS - search_file_info get ckline - fline=%s" % fline)
            tmp_filename = fmts['filename']['out'](fline.get('filename'))
            fdbg("FS - search_file_info ckname tmp_filename=%s" % tmp_filename)
            if tmp_filename == filename:
                fdbg("FS - search_file_info found newfn")
                fdbg("FS - search_file_info exit info=%s" % result)
                result = self._meta_data_loaded[index]
                break
            index = index + 1
        return result 

    # get filename from _meta_info and check if it exists in
    # _meta_data_loaded if so remove it from _meta_data_loaded
    def check_dup_metadata(self):
        rc = 0
        info = {}
        fdbg("FS - check_dup_metadata - _meta_info=%s" % self._meta_info)
        fdbg("FS - check_dup_metadata - _meta_data_loaded=%s" %
                self._meta_data_loaded)
        fmts = self._line_formats
        comps = self._line_components
        # get filename from _meta_info
        fline = self._meta_info.values()
        fdbg("FS - check_dup_metadata - fline=%s" % fline)
        info = fline[0]
        data = info
        fdbg("FS - check_dup_metadata - newfn data=%s" % data)
        newfn = fmts['filename']['out'](data.get('filename'))
        fdbg("FS - check_dup_metadata - newfn=%s" % newfn)
        # check for newfn in _meta_data_loaded
        index = 0
        for fline in self._meta_data_loaded.values():
            fdbg("FS - check_dup_metadata get ckline - fline=%s" % fline)
            info = fline
            data = info
            fdbg("FS - check_dup_metadata - oldfn data=%s" % data)
            fn = fmts['filename']['out'](data.get('filename'))
            fdbg("FS - check_dup_metadata ckname fn=%s" % fn)
            if fn == newfn:
                fdbg("FS - check_dup_metadata newfn duplicate - remove")
                del self._meta_data_loaded[index]
                rc = 1
            index = index + 1
        fdbg("FS - check_dup_metadata end - _meta_info=%s" % self._meta_info)
        fdbg("FS - check_dup_metadata end - _meta_data_loaded=%s" %
                self._meta_data_loaded)
        return rc

    def save_metadata(self):
        rc = 0
        warnstr = "Warning: Flash meta-data file ***do not edit***"
        meta_file = self._meta_file
        fdbg("FS - save_metadata - _meta_info=%s" % self._meta_info)
        fdbg("FS - save_metadata - _meta_info filename=%s" % meta_file)
        fdbg("FS - save_metadata - _meta_data_loaded=%s" %
                self._meta_data_loaded)
        if self._meta_info == {}:
            fdbg("FS - save_metadata - no _meta_info to save")
            fdbg("FS - save_metadata - mark empty")
            if self._meta_data_loaded == {}:
                fdbg("FS - save_metadata - no _meta_data_loaded to save")
                # no files, just write the warning string and push
                with open(meta_file, 'w') as f:
                    f.write("%s\n" % warnstr)
                verbose("No flash file meta info to save")
                try:
                    fdbg("FS - save meta files file_system_push")
                    rc = self.file_system_push()
                    if rc:
                        error("Failed to sync %s to EEPROM" % meta_file)
                        rc = 1
                except:
                    unhandled_exception()
                    error("Failed to save FLASH metadata to EEPROM")
                return rc
        fdbg("FS - save_metadata - mark 0 _meta_info=%s" % self._meta_info)
        if self._meta_info == {}:
            # delete operation just set _meta_info to meta_data_loaded
            self._meta_info = self._meta_data_loaded
        else:
            # combine meta_data_loaded with meta_info in meta_info
            mdlen = len(self._meta_data_loaded)
            self._meta_info[mdlen] = self._meta_info[0]
            self._meta_info.update(self._meta_data_loaded)
        fdbg("FS - save_metadata - mark 1 _meta_info=%s" % self._meta_info)
        # iterate through files (one per "line") in _meta_info
        # update metadata file with new metadata to save
        fdbg("FS - save_metadata - mark 2 keys=%s" % self._meta_info.keys())
        fdbg("FS - save_metadata - mark 3 values=%s" %
                self._meta_info.values())
        index = 0
        for fline in self._meta_info.values():
            fdbg("FS - save_metadata - fline=%s" % fline)
            info = fline
            meta_path = os.path.dirname(meta_file)
            fdbg("FS - save_metadata - mark 5")
            if not os.path.exists(meta_path):
                # Intentionally not using os.makedirs() for this
                error("Flash file system dir %s does not exist" % meta_path)
                return 1
            fmts = self._line_formats
            comps = self._line_components
            fdbg("FS - save_metadata - mark 8 fmts=%s" % fmts)
            data = info
            fdbg("FS - save_metadata - mark 10 data=%s" % data)
            fdbg("FS - save_metadata do writes to OneFS meta_file=%s" %
                    meta_file)
            try:
                if not index:
                    openarg = 'w'
                else:
                    openarg = 'a'
                with open(meta_file, openarg) as f:
                    fn = fmts['filename']['out'](data.get('filename'))
                    fdbg("FS - save_metadata do writes fn=%s" % fn)
                    sec = fmts['sector']['out'](data.get('sector'))
                    fdbg("FS - save_metadata do writes sec=%s" % sec)
                    fsz = fmts['filesize']['out'](data.get('filesize'))
                    fdbg("FS - save_metadata do writes fsz=%s" % fsz)
                    csz = fmts['compsize']['out'](data.get('compsize'))
                    fdbg("FS - save_metadata do writes csz=%s" % csz)
                    tstamp = fmts['timestamp']['out'](data.get('timestamp'))
                    fdbg("FS - save_metadata do writes tstamp=%s" % tstamp)
                    l = '%s : %s,%s,%s,%s' % (fn, sec, fsz, csz, tstamp)
                    fdbg("FS - save_metadata write to file l=%s" % l)
                    if not index:
                        # write warning line
                        f.write("%s\n" % warnstr)
                    f.write("%s\n" % l)
            except:
                unhandled_exception()
                error("Failed to save flash file system metadata to %s" %
                        meta_file)
                return 1
            index = index + 1
        try:
            fdbg("FS - save meta files file_system_push")
            rc = self.file_system_push()
            if rc:
                error("Failed to sync %s to EEPROM" % meta_file)
                rc = 1
        except:
            unhandled_exception()
            error("Failed to save flash file system metadata to EEPROM")
            return 1

        fdbg("FS - save_metadata end")
        return rc

    def file_system_push(self):
        rc = 0
        verbose("EEPROM update-push with local metadata files...")
        vpd = get_vpd_ef(options)
        err = vpd.update('push')
        if err:
            error("EEPROM update-push with local metadata files failed:", err)
        return err and 1 or 0

    def file_system_pull(self):
        rc = 0
        verbose("EEPROM update-pull with local metadata files...")
        vpd = get_vpd_ef(options)
        err = vpd.update('pull')
        if err:
            error("EEPROM update-pull with local metadata files failed:", err)
        return err and 1 or 0

    def display_file_system(self, sortkey='filename'):
        fdbg("FS - display_file_system")
        result = ['Flash file listing:']
        comps = self._line_components[:]
        formats = self._line_formats.copy()
        info = sorted(self._file_info.values(), key=lambda v: v.get(sortkey))
        fdbg("FS - display_file_system info=%s" % info)
        fdbg("FS - display_file_system _meta_info=%s" % self._meta_info)
        def comp_data(info, comp):
            data, format = info.get(comp), formats.get(comp)
            align, fmt = format.get('align'), format.get('format', lambda d:d)
            return (align, fmt(data), data)
        def info_format(info):
            if not info:
                return None
            sectors = len(info.get('sector', []))
            return ([comp_data(info, comp) for comp in comps] +
                    [(1, str(sectors), sectors)])
        data = filter(None, map(info_format, info))
        if not data:
            result.append("No flash files found")
            return result
        total = {
            'filename': "TOTAL SIZES", 'timestamp': '-' * len(time.ctime(0)),
            'filesize': 0, 'compsize': 0, 'sector': 0, 'count': 0,
            }
        fdbg("FS - display_file_system data=%s" % data)
        for line in data:
            fdbg("FS - display_file_system line=%s" % line)
            try:
                align, fmtd, d = zip(*line)
                if _fdbg:
                    print "FS - display_file_system align=", align
                    print "FS - display_file_system fmtd=", fmtd
                    print "FS - display_file_system d=", d
                sectors, compsize, filesize, ts, count = d[1:6]
                fdbg("FS - display_file_system d[0]=%s" % d[0])
                fdbg("FS - display_file_system d[1]=%s" % d[1])
                fdbg("FS - display_file_system d[2]=%s" % d[2])
                fdbg("FS - display_file_system d[3]=%s" % d[3])
                fdbg("FS - display_file_system d[4]=%s" % d[4])
                fdbg("FS - display_file_system d[5]=%s" % d[5])
                fdbg("FS - display_file_system filesize=%s" % filesize)
                fdbg("FS - display_file_system compsize=%s" % compsize)
                fdbg("FS - display_file_system sectors=%s" % sectors)
                fdbg("FS - display_file_system count=%s" % count)
                total['filesize'] += filesize
                total['compsize'] += compsize
                total['count'] += count
                fdbg("FS - display_file_system sector=%s" % sectors)
                fdbg("FS - display_file_system sectors max=%s" % max(sectors))
                fdbg("FS - display_file_system tsectors=%s" % total['sector'])
                if max(sectors) > total['sector']:
                    total['sector'] = max(sectors)
            except:
                unhandled_exception()
                pass
        sectors_width = max(map(len, map(str, [total['sector'], 'sector'])))
        total['sector'] = '-' * sectors_width
        comps.append("count"); formats["count"] = {'align': 1}
        hdrs = [(formats[c]['align'], c.capitalize(), c) for c in comps]
        total = [(formats[c]['align'], str(total[c]), c) for c in comps]
        data = [hdrs] + data + [total]
        widths = map(lambda d: max(map(lambda x: len(x[1]), d)), zip(*data))
        for line in data:
            a, f, d = zip(*line)
            comps = map(lambda (w, a, x): (a < 0 and '%-*s' or '%*s') % (w, x),
                        zip(widths, a, f))
            result.append("  ".join(comps))
        return result

    # create a directory tree for specified path
    def mkdirsfrompath(self, path):
        fdbg("FS - mkdirsfrompath make dirs for %s" % path)
        if not os.path.exists(path):
            l=[]
            p = "/"
            l = path.split("/")
            i = 1
            while i < len(l):
                p = p + l[i] + "/"
                i = i + 1
                fdbg("FS - mkdirsfrompath check path %s" % p)
                if not os.path.exists(p):
                    fdbg("FS - mkdirsfrompath dir=%s not present - make it" % p)
                    os.mkdir(p, 0755)
        if os.path.exists(path):
            fdbg("FS - mkdirsfrompath dir path %s created" % p)
        else:
            fdbg("FS - mkdirsfrompath dir path %s NOT created" % p)

class CMCFlashFileSystem(FlashFileSystem):

    def __init__(self, options={}):
        fdbg("CMCFS - init FS options=%s" % options)
        FlashFileSystem.__init__(self, options)

    def get_flash_size(self):
        if self._flash_size is None:
            self._flash_size = hal.getCMCFlashSize()
        fdbg("CMCFS - get_flash_size exit size=%s" % self._flash_size)
        return self._flash_size

    def get_flash_sector_size(self):
        if self._flash_sector_size is None:
            self._flash_sector_size = hal.getCMCFlashSectorSize()
        fdbg("CMCFS - get_flash_sector_size exit size=%s" %
                self._flash_sector_size)
        return self._flash_sector_size

    def _read(self, info, infile):
        fdbg("CMCFS - _read infile=%s info=%s" % (infile, info))
        # CMC flash offset = sector size * "virtual" sector number
        # int read_vpd_flash_cmc(char *filename, U32 sector, U32 xferlen)
        # => Returns num bytes read to filename, or -1 if error
        result, data = 0, None
        filename = infile
        sector_size = self.sector_size()
        # If compsize==0 then no compression was done
        txsize = info.get('compsize')
        fdbg("CMCFS - _read filename=%s compsize=%s" % (filename, txsize))
        if not txsize:
            txsize = info.get('filesize')
            fdbg("CMCFS - _read filesize=%s" % txsize)
        fdbg("CMCFS - _read txsize=%s" % txsize)
        if None in [filename, txsize]:
            error("%s has invalid flash file metadata" % filename)
            return None
        sectors = info.get('sector')
        fdbg("CMCFS - _read sectors=%s" % sectors)
        append = 0
        totalsize = 0
        seclistlen = len(sectors)
        startsec = min(sectors)
        seccnt = 0
        rlen = 0
        index = 0
        fdbg("CMCFS - _read - seclistlen=%s startsec=%s" %
                (seclistlen, startsec))
        for s in sectors:
            if s is None:
                error("%s cannot be read: no sectors specified" % filename)
                return None
            readnow = 0
            seccnt = seccnt + 1
            if seclistlen == 1:
                # just read the single sector
                readnow = 1
            else:
                if index == (seclistlen - 1):
                    # last sector, read now
                    readnow = 1
                else:
                    nextsect = sectors[(index + 1)]
                    fdbg("CMCFS - _read - s=%s nextsect=%s" % (s, nextsect))
                    if (s + 1) != nextsect:
                        # next sector not contigious, read what we have
                        readnow = 1
            # increase rlen by this full or partial sector to read
            if txsize > sector_size:
                plen = sector_size
            else:
                plen = txsize
            txsize -= plen
            rlen += plen
            if readnow:
                fdbg("CMCFS - _read - READ NOW - startsec=%s seccnt=%s" %
                    (startsec, seccnt))
                fdbg("CMCFS - _read txsize=%s rlen=%s" % (txsize, rlen))
                fdbg("CMCFS - _read append=%s" % append)
                # read data must be in contiguous flash sectors
                result = hal.hal_read_vpd_flash(filename, startsec,
                        seccnt, rlen, append)
                fdbg("CMCFS - _read result rlen=%s" % result)
                if result <= 0:
                    return None
                # adjust/set values from this read
                totalsize += rlen
                if not txsize:
                    # read last bytes, done
                    return totalsize
                # more to read, setup for next block
                append = 1
                # start next block with the following sector
                startsec = sectors[(index + 1)]
                seccnt = 0
                rlen = 0
            else:
                fdbg("CMCFS - _read - wait for more - startsec=%s seccnt=%s" %
                        (startsec, seccnt))
                fdbg("CMCFS - _read txsize=%s rlen=%s" % (txsize, rlen))
                fdbg("CMCFS - _read append=%s" % append)
            index = index + 1
        return totalsize

    def _write(self, info, outfile):
        fdbg("CMCFS - _write outfile=%s" % outfile)
        # CMC flash offset = sector size * "virtual" sector number
        # int write_vpd_flash_cmc(char *filename, U32 sector, U32 xferlen)
        # => Returns num bytes written to flash, or -1 if error
        # input file name "filename" from which we read data to write to flash
        # input "virtual" sector to start write to
        # input number of bytes to read from the file and write to flash
        # Routine will first erase the desired "sector(s)" to be written
        # length to erase with be number of sectors * sector size
        sector_size = self.sector_size()
        # If compsize==0 then no compression was done
        txsize = info.get('compsize')
        if not txsize:
            txsize = info.get('filesize')
        fdbg("CMCFS - _write txsize=%s" % txsize)
        filename = outfile
        fstat = os.stat(outfile)
        fsize = fstat.st_size
        if not fsize or txsize != fsize:
            error("Invalid flash write file size %s" % fsize)
            return None
        fdbg("CMCFS - _write filename=%s" % filename)
        fdbg("CMCFS - _write info=%s" % info)
        sectors = info.get('sector')
        if None in [filename, txsize, sectors]:
            error("%s has invalid flash file metadata" % filename)
            return None
        skipbytes = 0
        totalsize = 0
        seclistlen = len(sectors)
        startsec = min(sectors)
        seccnt = 0
        wlen = 0
        index = 0
        fdbg("CMCFS - _write - seclistlen=%s startsec=%s" %
            (seclistlen, startsec))
        for s in sectors:
            if s is None:
                error("%s cannot be written: flash is full" % filename)
                return None
            writenow = 0
            seccnt = seccnt + 1
            if seclistlen == 1:
                # just write the single sector
                writenow = 1
            else:
                if index == (seclistlen - 1):
                    # last sector, write now
                    writenow = 1
                else:
                    nextsect = sectors[(index + 1)]
                    fdbg("CMCFS - _write - s=%s nextsect=%s" % (s, nextsect))
                    if (s + 1) != nextsect:
                        # next sector not contigious, write what we have
                        writenow = 1
            # increase wlen by this full or partial sector to write
            if txsize > sector_size:
                plen = sector_size
            else:
                plen = txsize
            txsize -= plen
            wlen += plen
            if writenow:
                fdbg("CMCFS - _write - WRITE NOW - startsec=%s seccnt=%s" %
                        (startsec, seccnt))
                fdbg("CMCFS - _write txsize=%s wlen=%s" % (txsize, wlen))
                fdbg("CMCFS - _write skipbytes=%s" % skipbytes)
                # write data must be in contiguous flash sectors
                result = hal.hal_write_vpd_flash(filename, startsec,
                        seccnt, wlen, skipbytes)
                fdbg("CMCFS - _write result wlen=%s" % result)
                if result <= 0:
                    return None
                # adjust/set values from this write
                totalsize += wlen
                if not txsize:
                    # wrote last bytes, done
                    return totalsize
                # more to write, setup for next block
                skipbytes += wlen
                # start next block with the following sector
                startsec = sectors[(index + 1)]
                seccnt = 0
                wlen = 0
            else:
                fdbg("CMCFS - _write - wait for more - startsec=%s seccnt=%s" %
                        (startsec, seccnt))
                fdbg("CMCFS - _write txsize=%s wlen=%s" % (txsize, wlen))
                fdbg("CMCFS - _write skipbytes=%s" % skipbytes)
            index = index + 1
        return totalsize

# ############## #
# FLASH COMMANDS #
# ############## #

def init_f(options):
    fdbg("FS - init_f")
    # optionally have a flag to force no prompt;
    verbose("Initializing flash files data...")
    fdbg("FS - init_f call get_flash_file_system options=%s" % options)
    flash = get_flash_file_system(options)
    return not flash and 1 or 0

def check_f(options):
    fdbg("FS - check_f")
    vpd = get_vpd_ef(options)
    verbose("Checking flash file data...")
    err = get_flash_file_system(options).check_data()
    if err:
        error("Flash file data invalid: Consider re-init?:", err)
    else:
        verbose("Flash file data appears valid!")
    return err and 1 or 0

def dump_f(options):
    fdbg("FS - dump_f")
    vpd = get_vpd_ef(options)
    # ASSUMES that necessary EEPROM VPD file syncing has been done --
    # and that the local /etc/isilon_flash_files "cache" file is available
    flash = get_flash_file_system(options)
    if not flash:
        return 1
    for line in flash.display_file_system():
        print line
    return 0

def flash_read(options):
    fdbg("FS - flash_read")
    vpd = get_vpd_ef(options)
    filename = options.get('filename')
    if not filename:
        print "Supply filename to read"
        return 1
    verbose("Reading flash file: %s" % filename)
    return get_flash_file_system(options).read(filename)

def flash_delete(options):
    fdbg("FS - flash_delete")
    vpd = get_vpd_ef(options)
    filename = options.get('filename')
    verbose("Deleting flash file: %s" % filename)
    if not filename:
        return 1
    result = get_flash_file_system(options).delete(filename)
    return result is None and 1 or 0

def flash_write(options):
    fdbg("FS - flash_write")
    vpd = get_vpd_ef(options)
    filename = options.get('filename')
    verbose("Writing flash file: %s" % filename)
    if not filename:
        return 1
    result = get_flash_file_system(options).write(filename)
    return result is None and 1 or 0

flashfilesinitdesc = ("Init %s with default CTO files" %
        files.node_flash_data_file())
flash_cmds_list = [
    #
    # INIT & CHECK COMMANDS
    #
    ('finit',
    { 'func': flash_init,
      'title': 'Flash Init to latest version (destructive)',
      'flash': True,
      }),
    ('finitrev',
    { 'func': flash_initrev,
      'title': 'Flash Init with given version (destructive)',
      'args' : ['version'],
      'flash': True,
      }),
    ('finitcto',
    { 'func': flash_initcto,
      'title': flashfilesinitdesc,
      'flash': True,
      }),
    ('fcheck',
    { 'func': flash_check,
      'title': 'Flash Data Check (validate header, data)',
      'flash': True,
      }),
    #
    # DONGLE SYNC SUPPORT COMMANDS: PUSH, PULL, COPY, RESTORE, TMP
    #
    ('fpush',
    { 'func': flash_push,
      'title': 'Push data - to Flash from local files',
      'flash': True,
      }),
    ('fpull',
    { 'func': flash_pull,
      'title': 'Pull data - from Flash to local files',
      'flash': True,
      }),
    ('fcopy',
    { 'func': flash_copy,
      'title': 'Copy CTO data - Flash written with CTO files and checksums',
      'flash': True,
      }),
    ('frestore',
    { 'func': flash_restore,
      'title': 'Restore CTO data - CTO files restored from Flash',
      'flash': True,
      }),
    ('ftmp',
    { 'func': flash_tmp,
      'title': 'Restore CTO data - CTO files restored from Flash to /tmp',
      'flash': True,
      }),
    ('ftmp_md5',
    { 'func': flash_tmp_md5,
      'title': 'Restore CTO data - CTO checksums restored from Flash to /tmp',
      'flash': True,
      }),
    #
    # DUMP COMMANDS
    #
    ('fdump',
    { 'func': flash_dump,
      'title': 'Dump Flash header - file listing',
      'flash': True,
      }),
    ('fdumphdr',
    { 'func': flash_dumphdr,
      'title': 'Dump Flash header - formatted',
      'flash': True,
      }),
    ('fdumpraw',
    { 'func': flash_dumprawhdr,
      'title': 'Dump Flash header - raw data, at given offset + length',
      'args': ['off','len'],
      'raw':True,
      'flash': True,
      }),
    ('fdumpall',
    { 'func': flash_dumprawhdr,
      'title': 'Dump Flash header - raw data, ALL (long!)',
      'raw':True,
      'flash': True,
      }),
    #
    # READ & WRITE COMMANDS
    #
    ('fread',
    { 'func': flash_read,
      'title': 'Read Flash - read given filename (search)',
      'args': ['filename'],
      'flash': True,
      }),
    ('fdelete',
    { 'func': flash_delete,
      'title': 'Write Flash - delete given filename',
      'args': ['filename'],
      'flash': True,
      }),
    ('fwrite',
    { 'func': flash_write,
      'title': 'Write Flash - write given filename',
      'args': ['filename'],
      'flash': True,
      }),
    ]


def verbose_func():
    def func(x):
        global _verbose
        if _verbose is True or _verbose is False:
            # Either True or False - same thing, start at 1
            _verbose = 1
        elif isinstance(_verbose, int):
            _verbose += 1
        else:
            raise Exception("Invalid _verbose value: %s" % _verbose)
    return func

# ############ #
# MAIN & USAGE #
# ############ #

if _node_has_flash:
    cmds_list = cmds_list + flash_cmds_list
if not _node_has_resume:
    cmds_list = [(c,d) for (c,d) in cmds_list if not d.get('resume')]
if _group_flash_cmds:
    _cmds_with_flash = filter(lambda (c,x): not x.get('flash'), cmds_list)
    _cmds_wo_flash = filter(lambda (c,x): x.get('flash'), cmds_list)
    cmds_list = _cmds_with_flash + _cmds_wo_flash
commands = [ c for (c,x) in cmds_list ]
cmds_desc = dict(cmds_list)
fmt_cmdopt = lambda c: (cmds_desc[c].has_key('args') and
                     reduce(lambda a,b:a+' '+b, cmds_desc[c]['args'])
                     or '')
fullcmdslist = map(lambda c:"%s %s" % (c, fmt_cmdopt(c)), commands)
fullcmds = dict(zip(commands, fullcmdslist))
argwidth = 1 + max(map(lambda x:len(x), fullcmdslist))
fmt_opt = lambda (o,d): "  %-*s: %s" % (argwidth,o,d)
fmt_cmd = lambda c: "  %-*s: %s" % (argwidth, fullcmds[c], cmds_desc[c]['title'])
disp_opts = reduce(lambda a,b: a+"\n"+b, map(fmt_opt, opts_desc))
disp_cmds = reduce(lambda a,b: a+"\n"+b, map(fmt_cmd, commands))

def main(options):
    global _debug, _verbose, _force, _single, _fdbg, _devid

    fdbg("main - options=%s" % options)
    result = 0
    show_usage = False
    args = argslib.Args(options)
    non_opts = args.process(sys.argv[1:])
    usercmds = non_opts
    fdbg("main - usercmds=%s" % usercmds)
    fdbg("main - _node_has_flash=%s" % _node_has_flash)
    fdbg("main - _group_flash_cmds=%s" % _group_flash_cmds)
    if len(usercmds) == 0:
        raise argslib.Error('No command specified')

    if args['fdbg']:
        _fdbg = True
    if args['debug']:
        _debug = True
    if args['quiet']:
        _verbose = False
    if args['force']:
        _force = True
    if args['single']:
        _single = True
    if args['devid'] is not None:
        _devid = args['devid']

    vpdoptions = {
        'verbose': _verbose,
        'debug': _debug,
        'unit_eeprom_test': args['unittest'],
        }

    # Initializing the vpd automatically upgrades it.  If we're going to
    # initialize the EEPROM after, might as well skip it.
    vpdoptions['skip_upgrade'] = 'init' in usercmds or 'initrev' in usercmds

    # If only using "raw" commands skip the init read of EEPROM header rev
    actualcmds = filter(lambda c: c in commands,
                        [str(c).lower() for c in usercmds])
    rawcases = [ cmds_desc.get(c).get('raw') for c in actualcmds ]
    rawonly = len(filter(lambda x: x!=True, rawcases)) == 0
    if rawonly:
        vpdoptions['raw'] = True
    # dumppbb() is a hidden command
    if 'dumppbb' in usercmds:
        actualcmds.append('dumppbb')
    if len(actualcmds) == 0:
        raise argslib.Error('No valid command specified')

    while len(usercmds):

        valid, func_options = True, {}
        cmd = str(usercmds.pop(0)).lower()
        if cmd == 'dumppbb':
            if dumppbb():
                result, valid = 1, False
            continue
        cmddict = cmds_desc.get(cmd, {})
        cmdfunc = cmddict.get('func')
        cmdargs = cmddict.get('args')
        argfunc = cmddict.get('argfunc', generic_get_args)
        if not cmdfunc:
            error("No valid function found for cmd '%s'" % cmd)
            result, valid, show_usage = 1, False, True
        elif cmdargs:
            valid, func_options, usercmds = argfunc(usercmds, cmdargs)
            if not valid:
                error("Insufficient or Invalid args for cmd '%s'" % cmd)
                result = 1
                show_usage = True

        if valid:
            func_options['vpd'] = vpdoptions
            options['vpd'] = vpdoptions
            try:
                if isinstance(cmd, str):
                    fcmd = cmd[0]
                    fdbg("FS - main func_options=%s" % func_options)
                    fdbg("main - cmd=%s fcmd=%s" % (cmd, fcmd))
                    if fcmd not in [ 'f', 'F' ]:
                        vpd = get_vpd(options, cmd)
                        fdbg("main - nonflash cmd vpd=%s" % vpd)
                        result = cmdfunc(vpd, func_options)
                    else:
                        fdbg("main - flash cmd")
                        result = cmdfunc(func_options)
                else:
                    error("Invalid command '%s'" % cmd)
                    result = 1
            except IOError, e:
                error("IOError during '%s':" % cmd, e)
                result, valid = 1, False

    if show_usage:
        raise argslib.Error('Invalid command usage')

    fdbg("main - exit result=%s" % result)
    return result

if __name__ == '__main__':
    usage = \
"""Usage: %s [OPTIONS] command [commands]

OPTIONS:
%s

COMMANDS:
%s

Isilon %s Version %s %s.
Provides Init, Check, Push, Pull, Sync, Dump, Read, Write commands.
Legacy VPD files listed in /etc/ifs/dongle_sync_files
Multiple commands are allowed.  Sample:
%s write a 23 writen 2 foo bar keyB valB dumphdr dump read foo
""" % (argslib.appname(), disp_opts, disp_cmds, TITLE, VERSION, REVISION,
           argslib.appname())

    options = {

        'fdbg':
        {'opts'    : ['z', 'fdbg'],
         'default' : False,
         'func'    : argslib.true_func(),
         },

        'single':
        {'opts'    : ['s', 'single'],
         'default' : False,
         'func'    : argslib.true_func(),
         },

        'force':
        {'opts'    : ['f', 'force'],
         'default' : False,
         'func'    : argslib.true_func(),
         },

        'debug':
        {'opts'    : ['d', 'debug'],
         'default' : False,
         'func'    : argslib.true_func(),
         },

        'verbose':
        {'opts'    : ['v', 'verbose'],
         'default' : False,
         'func'    : verbose_func(),
         },

        'devid':
        {'opts'    : ['devid='],
         'default' : None,
         'func'    : str, # Really an int, but support 'all'
         },

        'quiet':
        {'opts'    : ['q', 'quiet'],
         'default' : False,
         'func'    : argslib.true_func(),
         },

        'unittest':
        {'opts'    : ['unittest'],
         'default' : False,
         'func'    : argslib.true_func(),
         },

        'help':
        {'opts'    : ['h', 'help'],
         'func'    : argslib.help_func(usage),
         },
        }

    sys.exit(argslib.run(lambda :main(options), usage))
