#!/usr/bin/python
# ex: set shiftwidth=4 tabstop=4 expandtab:

# This version has much work on extending isi_gather_info to handle generic
# info,file,script output -- Indeed, anything cluster-wide should be possible
# if it can be encapsulated into a provided node-run script.
#
# Be sure to maintain backwards compatibility with Releases 3.0, 3.5, 4.0.
# In absence of new options, all behavior should be identical to previous
# isi_gather_info versions.
#
# This script will use /ifs by default to store results.  This is a
# change from previous behavior and was requested explicitly from
# support.
#
# Selected Options Info:
#  o Filenames:
#    Gathered just like Required Logs: Will handle both Dirs and Files.
#  o gather_script, gather_expr:
#    Will be added to the generated Bash script with UTILITIES.
#    Output will be piped to an appropriate named file, based on generated
#    command.
#  o analysis_script:
#    Will be executed before tarball generation.
#    Output will be piped to an appropriate named file.
#  o tarfile:
#    Replaces the auto-generated pkg_path_name.
#    If full path specified, use that. Else, use pkg_path_name code for path.
#  o nologs:
#    Only way to skip gathering required logs.

import ftplib
import getopt
import getpass
import os
import random
import re
import signal
import string
import sys
import tarfile
import tempfile
import time
import threading
import thread
import traceback
import httplib
import subprocess

import socket
import isi.config.array as array
import isi.config.arrcfg as arrcfg
import isi.config.licensing as licensing
import isi.net.rpc as rpc
import isi.gconfig as gconfig

import isi.app.lib.cluster
import isi.app.lib.procs as procs
import isi.fs.restripe.group as group
from isi.config import mxd
from isi.ui.lib.guiformat import format_capacity
from isi.misc import sysctlbyname

import shutil

# for package xml metadata
import xml.etree.ElementTree as etree
import isi.celog.attachments
import isi.celog.celog_config_api
import isi.config.arrcfg

# for ESRS v2 (REST) transactions
import isi.esrs.restclient as esrs

PROGNAME = os.path.basename(sys.argv[0])
# Bug 167340 - switch to git attr ident
REVISION = '$Id: 19051e6cfd16c5715af42e123b522f8b00636479 $'


SUPPORT_DIR           = None
TMP_SUPPORT_DIR       = None
LOCAL_TMP_SUPPORT_DIR = None
PKG_SUPPORT_DIR       = None
RUN_DIR               = None
GATHER_SCRIPT         = None
ISI_NODES_OUTPUT      = None
ISI_FIRMWARE_OUTPUT   = None
PACKAGE_INFO          = None
CELOG_EVENTS_INFO     = None
SAVED_COMMAND_LINE    = None
GATHER_RUN_FILE       = None
# used to lock and write out the node ID in ifsvar.
GATHER_NODE_FD        = None
IFS_READONLY          = False

_TMP_SUPPORT_DIR       = 'tmp/'
_PKG_SUPPORT_DIR       = 'pkg/'
_GATHER_SCRIPT         = '_gather'
_LOCAL_GATHER_SCRIPT   = '_local_gather'
_LOCAL_TMP_SUPPORT_DIR = "local"
_ISI_NODES_OUTPUT      = 'nodes_info'
_ISI_FIRMWARE_OUTPUT   = 'firmware'
_PACKAGE_INFO          = 'package_info.xml'
_CELOG_EVENTS_INFO     = 'celog_events.xml'
_SAVED_COMMAND_LINE    = 'command_line'
_INCREMENTAL_FILE      = 'last_gather'
_FULLGATHER_FILE       = 'last_full_gather'

VAR_SUPPORT_DIR = '/var/crash/support/'
IFS_SUPPORT_DIR = '/ifs/data/Isilon_Support/'
RUN_SUPPORT_DIR = os.path.join(os.path.sep + 'ifs', '.ifsvar',
        'run' + os.path.sep)
RUN_VAR_SUPPORT_DIR = os.path.join(VAR_SUPPORT_DIR, 'run'
         + os.path.sep + 'tmp' + os.path.sep)

GATHER_STATUS_PATH  = '/ifs/.ifsvar/run/gather-status'
GATHER_STATUS_LOCK  = GATHER_STATUS_PATH + '.lock'
GATHER_NODE_FILE_PATH = '/ifs/.ifsvar/run/gather.node'

CONFIGDIR = '/etc/mcp/'
TIMESTAMP_COMMAND = "python -c 'import timeit; print timeit.default_timer();'"

# 'exe': cmd, name: cmd==command, name==output filename
# 'tar': cmd, name: cmd==cd to parent dir; name==tarball_filename tarball_dir
# See generate_scripts for the consumer of this information.
UTILITIES = {
    'abr_config' : ('exe', "${SUDO}/usr/bin/isi_make_abr &>/dev/null; "
                                "${SUDO}/usr/bin/isi_inventory_tool "
                                "--display --itemType=asbuilt", "asbuilt.xml"),
    'aspera_config' : ('local_tar', "cd /", "aspera_config.tar " \
                                 "/ifs/.ifsvar/aspera/*"),
    'celog_node_db'   : ('tar', "cd /", "celog_db.tar /var/db/celog_* "),
    'celog_ifsvar_db' : ('local_tar', "cd /", "celog_ifsvar_db.tar " \
                                "/ifs/.ifsvar/db/celog*"),
    'configs'         : ('tar', "cd /", "local_configs.tar "    \
                                "etc/celog "                  \
                                "etc/mcp "                  \
                                "etc/isilon_serial_number " \
                                "etc/isilon_system_config "),
    'isi_phone_home_config_local' : ('local_tar', "cd /", "iph_config_local.tar " \
                                                "/ifs/.ifsvar/run/phone_home.disabled " \
                                                "/ifs/.ifsvar/phone_home/config.json "),
    'isi_phone_home_config_node'  : ('tar', "cd /", "iph_config_node.tar " \
                                                "/usr/local/isi_phone_home/utils/config.json "),
    'cto_summary'     : ('exe', "${SUDO}/usr/bin/isi_cto_summary",
                                "cto_summary"),
    'dedupe_reports'  : ('local_exe', "/usr/bin/isi dedupe reports list -v",
                                      "dedupe_reports"),
    'dedupe_settings' : ('local_exe', "/usr/bin/isi dedupe settings view",
                                      "dedupe_settings"),
    'dedupe_stats'    : ('local_exe', "/usr/bin/isi dedupe stats",
                                      "dedupe_stats"),
    'dev.ioat'        : ('exe', "/sbin/sysctl dev.ioat", "dev.ioat"),
    'dev.ntb_hw.0'    : ('exe', "/sbin/sysctl dev.ntb_hw.0", "dev.ntb_hw.0"),
    'sfm.shadow.stats': ('exe', '/sbin/sysctl efs.sfm.shadow.stats',
                                'efs.sfm.shadow.stats'),
    'sfm.clone.stats': ('exe', '/sbin/sysctl efs.sfm.clone.stats',
                                'efs.sfm.clone.stats'),
    'sfm.dedupe.stats': ('exe', '/sbin/sysctl efs.sfm.dedupe.stats',
                                'efs.sfm.dedupe.stats'),
    'compliance'      : ('local_exe', "/sbin/sysctl isi.compliance", "isi_compliance"),
    'df'              : ('exe', "/bin/df -li", "df"),
    'du'              : ('exe', "/usr/bin/du -hd3 /var", "du"),
    'etcifs'          : ('tar', "cd /etc", "etcifs.tar ifs"),
    'etc'             : ('tar', "cd /", "etc.tar --exclude '*master\.passwd' etc"),
    'efs.gmp.group'   : ('exe', '/sbin/sysctl efs.gmp.group', 'efs.gmp.group'),
    'efs.l3'          : ('exe', '/sbin/sysctl efs.l3', 'efs.l3'),
    'efs.lbm.drive_space':
                        ('exe', '/sbin/sysctl efs.lbm.drive_space',
                                'efs.lbm.drive_space'),
    'efs.snapshot'    : ('exe', '/sbin/sysctl efs.snapshot', 'efs.snapshot'),
    'efs.stf'         : ('exe', '/sbin/sysctl efs.stf', 'efs.stf'),
    'fstat'           : ('exe', "/usr/bin/fstat", "fstat"),
    'gconfig_local'   : ('tar', "cd /", "gconfig_local.tar " \
                                "/var/ifs/*.gc " \
                                "/etc/ifs/*.gc"),
    'gconfig_ifsvar'  : ('local_tar', "cd /", "gconfig_ifsvar.tar " \
                                "/ifs/.ifsvar/main_config*" ),
    'history'         : ('tar', "cd /", "history.tar /var/ifs/history" ),
    'zsh_history'     : ('tar', "cd /root", "zsh_history.tar .zsh_history" ),
    'zsh_history_compadmin'  : ('tar', "cd /compadmin", "zsh_history_compadmin.tar .zsh_history" ),
    'kern.cam.ada'          : ('exe', "/sbin/sysctl kern.cam.ada", "kern.cam.ada"),
    'hw.disk_event'   : ('exe', "/sbin/sysctl hw.disk_event", "hw.disk_event"),
    'hw.ecclist'      : ('exe', "/sbin/sysctl hw.ecclist", "hw.ecclist"),
    'hw.iosched'       : ('exe', "/sbin/sysctl hw.iosched", "hw.iosched"),
    'hw.mnv0'         : ('exe', "/sbin/sysctl hw.mnv0", "hw.mnv0"),
    'hw.mnvi'         : ('exe', "/sbin/sysctl hw.mnvi", "hw.mnvi"),
    'ifconfig'        : ('exe', "/sbin/ifconfig -a", "ifconfig"),
    'ifsvar_db_pkg'   : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_db_pkg.tar "\
                                      "--exclude '*.tar' db/pkg"),
    'ifsvar_db_pkg_tars': ('local_exe',
                            "/usr/bin/find /ifs/.ifsvar/db/pkg -name '*.tar' "\
                            "-ls -exec /usr/bin/tar -tvf {} \;",
                            "ifsvar_db_pkg_tars"),
    'ifsvar_etc'      : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_etc.tar etc"),
    'ifsvar_modules'  : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_modules.tar "
                         "--exclude stats --exclude bedrs/dedupe_metadata "
                         "--exclude bedrs/dry_run_metadata "
                         "--exclude 'fsa/pub' --exclude 'fsa/work' "
                         "--exclude 'tsm/samba-bak' "
                         "--exclude 'hardware/drives/firmware' "
                         "--exclude 'jobengine/cp' --exclude tardis "
                         "--exclude 'isi_vol_copy/[0-9]*/dumpstream\.[0-9]*' modules"),
    'ifsvar_modules_stats'  : ('local_tar', "cd /ifs/.ifsvar", \
                               "ifsvar_modules_stats.tar " \
                                "modules/stats"),
    'ifsvar_modules_fsa'  : ('local_tar', "cd /ifs/.ifsvar", \
                               "ifsvar_modules_fsa.tar " \
                                "modules/fsa/pub"),
    'ifsvar_modules_jobengine_cp' : ('local_tar', "cd /ifs/.ifsvar/modules/jobengine", \
                               "ifsvar_modules_jobengine_cp.tar " \
                                "cp"),
    'ifsvar_modules_tardis' : ('local_tar', # force a bkp and then archive them
                               'cd /ifs/.ifsvar && ${SUDO}/usr/sbin/isi_tardis_backup now',
                               'ifsvar_modules_tardis.tar '
                               # exclude backup lock file
                               '--exclude modules/tardis/backups/bkp.lck '
                               # collect everything, even including the
                               # live namespaces as isi_tardis_backup might
                               # fail if IFS is readonly or for any other error
                               # and we may not capture all the latest copies
                               # SEE: Bug 159882
                               'modules/tardis'),
    'ifsvar_tmp'      : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_tmp.tar tmp" ),
    'ifsvar_smb'      : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_smb.tar smb" ),
    'ifsvar_upgrade'  : ('local_tar', 'cd /ifs/.ifsvar', 'ifsvar_upgrade.tar '
                         # exclude the archive dir entirely
                         '--exclude upgrade/archive '
                         'upgrade'),
    'licensing_ifsvar': ('local_tar', "cd /", "licensing_ifsvar.tar " \
                         "/ifs/.ifsvar/licensing*.log" ),
    'upgrade_archive' : ('local_tar', 'cd /ifs/.ifsvar/upgrade',
                         'upgrade_archive.tar archive'),
    'upgrade_local'   : ('tar', 'cd /', 'upgrade_local.tar '
                         # exclude the sqlite db and journal files
                         '--exclude /var/ifs/upgrade/AgentPersistent.db* '
                         '/var/ifs/upgrade'),
    'upgrade_agent_persistent_db' : ('exe',
             # backup the sqlite-db to a temporary file
             '${SUDO}/usr/local/bin/sqlite3 /var/ifs/upgrade/AgentPersistent.db .dump',
             'upgrade_agent_persistent_db.sql'),
    'isi_upgrade_archive_gather' : ('local_exe',
                                    "${SUDO}/usr/bin/isi_upgrade_archive_gather -puf",
                                    "upgrade_archive_gather.json"),
    # this utility takes long time.
    'isi_bug_info'    : ('exe', "${SUDO}/usr/bin/isi_bug_info", "isi_bug_info"),
    'isi_devices_drive_list'  : ('exe', "${SUDO}/usr/bin/isi devices drive list", "isi_devices_drive_list"),
    'isi_devices_firmware_list': ('exe', "${SUDO}/usr/bin/isi devices firmware list", "isi_devices_firmware_list"),
    'isi_devices_firmware_summary': ('local_exe', "${SUDO}/usr/bin/isi devices firmware list --summary", "isi_devices_firmware_summary"),
    'isi_drivenum'    : ('exe', "${SUDO}/usr/bin/isi_drivenum -A", "isi_drivenum"),
    'isi_disk_inject' : ('exe', '${SUDO}/usr/bin/isi_hwtools/isi_disk_inject -l',
                                'isi_disk_inject'),
    'isi_diskpool'    : ('local_exe', "/usr/bin/disi -I diskpool list -v",
                                "isi_diskpool"),
    'efs.bam.disk_pool_db':
                        ('exe', "/sbin/sysctl efs.bam.disk_pool_db",
                                    'efs.bam.disk_pool_db'),
    'isi_event_events'      : ('local_exe', "${SUDO}/usr/bin/isi event events list", "isi_event"),
    'isi_statistics_drive'  : ('exe', "${SUDO}/usr/bin/isi statistics drive " \
                                      "--degraded --long", "isi_statistics_drive"),
    'isi_statistics_heat': ('local_exe', "${SUDO}/usr/bin/isi statistics heat " \
                                      "--degraded --long", "isi_statistics_heat"),
    'isi_dmi_log'     : ('exe', "${SUDO}/usr/bin/isi_gather_memory_ecc_info", "isi_dmilog"),
    'isi_hw_status'   : ('exe', "${SUDO}/usr/bin/isi_hw_status -Lcisfvtbg",
                                "isi_hw_status"),
    'isi_ib_bug_info' : ('exe', "${SUDO}/usr/bin/isi_ib_bug_info", "isi_ib_bug_info"),
    'isi_license'     : ('local_exe', "/usr/bin/isi license list", "isi_license"),
    'isi_license.xml' : ('local_exe', "/bin/cat /ifs/.ifsvar/isi_license.xml", "isi_license.xml"),
    'isi_esrs'        : ('local_exe', "(${SUDO}/usr/bin/isi_gconfig -t esrs" \
                                      "|grep -v 'api_key'; echo;" \
                                      "${SUDO}/usr/bin/grep -v 'api_key'" \
                                      " /ifs/.ifsvar/esrs2_config_changes.log)",
                                      "isi_esrs"),
    'isi_remote_support' : ('local_exe', "${SUDO}/usr/bin/isi_gconfig"\
                                         " -t remote-support",
                                         "isi_remote_support"),
    # It's possible that user create lots of exports
    'isi_nfs_exports' : ('local_exe', "/usr/bin/isi_gather_nfs_exports",
                                "isi_nfs_exports"),
    'isi_nfs_aliases' : ('local_exe', "/usr/bin/isi nfs aliases list",
                                "isi_nfs_aliases"),
    'isi_psi_tool_receipt' : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_psi_tool -v",
                                     "isi_psi_tool_receipt"),
    #MAV 7.0 CLI update
    'isi_smb'         : ('local_exe', "${SUDO}/usr/bin/isi_gather_smb_info", "isi_smb"),
    'isi_stat'        : ('local_exe', "${SUDO}/usr/bin/isi stat -a", "isi_stat"),
    'isi_stat-p'      : ('local_exe', "${SUDO}/usr/bin/isi stat -p -v", "isi_stat-p"),
    'isi_status'      : ('exe', "${SUDO}/usr/bin/isi status", "isi_status"),
    'isi_upgrade_view': ('local_exe', "${SUDO}/usr/bin/isi upgrade cluster view",
                                "isi_upgrade_view"),
    'isi_upgrade_list': ('local_exe', "${SUDO}/usr/bin/isi upgrade cluster list",
                                "isi_upgrade_list"),
    'isi_sync_policy' : ('local_exe', "${SUDO}/usr/bin/isi sync pol list -v",
                                "isi_sync_policy"),
    'isi_ufp'         : ('exe', "/usr/bin/isi_ufp -a", "isi_ufp"),
    'isi_usysctl_psi' : ('exe', "/usr/bin/isi_usysctl -m psi", "isi_usysctl_psi"),
    'kern.geom'       : ('exe', '/sbin/sysctl kern.geom',
                                'kern.geom'),
    'kern.proc.all_stacks' :
                        ('exe', '/sbin/sysctl kern.proc.all_stacks',
                                'kern.proc.all_stacks'),
    'kern.openfiles'  : ('exe', '/sbin/sysctl kern.openfiles',
                                'kern.openfiles'),
    'kern.mbaudit'    : ('exe', '/sbin/sysctl kern.mbaudit', 'kern.mbaudit'),
    'kern.malloc_pigs': ('exe', "/sbin/sysctl kern.malloc_pigs",
                                "kern.malloc_pigs"),
    'netstat'         : ('exe', "(/usr/bin/netstat -sn; echo; " \
                                "/usr/bin/netstat -rn; echo; " \
                                "/usr/bin/netstat -lan; echo; " \
                                "/usr/bin/netstat -m; echo; " \
                                "/usr/bin/netstat -B; echo; " \
                                "/usr/bin/netstat -in; echo; " \
                                "/usr/bin/netstat -Q; echo; " \
                                "/usr/bin/netstat -rs)", "netstat"),
    'node_info_cache' : ('local_tar', "cd /", "node_info_cache.tar " \
                                "/ifs/.ifsvar/db/node_info_cache"),
    'isi_nfs4mgmt'    : ('exe', "/usr/bin/isi_nfs4mgmt", "isi_nfs4mgmt"),
    'nfsstat'         : ('exe', '/usr/bin/isi_gather_nfsstat', 'nfsstat'),
    'override'        : ('tar', "cd /etc/mcp", "override.tar override"),

    # get the list of installed patches and backup the patch system databases
    'patch_list'        : ('local_exe',
                           "/usr/bin/isi upgrade patches list -v", "patch_list"),
    'patch_list_local'  : ('exe',
                           "/usr/bin/isi upgrade patches list -v --local",
                            "patch_list_local"),
    'patch_installed_db': ('exe',
                '${SUDO}/usr/local/bin/sqlite3 /var/patch/installed.db .dump',
                'patch_installed_db.sql'),
    'patch_activity_db' : ('exe',
                '${SUDO}/usr/local/bin/sqlite3 /var/patch/activity.db .dump',
                'patch_activity_db.sql'),
    'patch_cluster_db'  : ('local_exe',
                '${SUDO}/usr/local/bin/sqlite3 /ifs/.ifsvar/patch/patch.db .dump',
                'patch_cluster_db.sql'),

    'rpcinfo'         : ('exe', "/usr/bin/rpcinfo", "rpcinfo"),
    'vfs.nfs'         : ('exe', "/sbin/sysctl vfs.nfs", "vfs.nfs"),
    'vfs.nlm'         : ('exe', "/sbin/sysctl vfs.nlm", "vfs.nlm"),
    'vm.uma_zalloc_pigs' : ('exe', "/sbin/sysctl vm.uma_zalloc_pigs",
                                   "vm.uma_zalloc_pigs"),
    # Bug 129137 - journal stats for log analysis to see if some code paths
    # can be simplified.
    'journal_stats'   : ('exe', "/sbin/sysctl " \
                                "efs.journal.stats.write.overwrite " \
                                "efs.journal.stats.write.restarted",
                                "journal_stats"),

    # Don't run quota list on every node.
    #MAV 7.0 - this is being changed to isi_classic until the command:
    #isi quota quota list --format json is suitable to replace it
    'quotas'          : ('local_exe', "/usr/bin/isi_classic quota ls --export",
                                "isi_quota"),
    #MAV 7.0 CLI update
    'quotas_verbose'  : ('local_exe', "/usr/bin/isi quota quotas list  --verbose --format list",
                                "isi_quota_verbose"),
    'sockstat'        : ('exe', "/usr/bin/sockstat", "sockstat"),
    'sin_cache'       : ('exe', "/sbin/sysctl debug.dump_sfm_sin_cache",
                                "sin_cache"),
    'sstore_list'     : ('local_exe', "/usr/sbin/isi_sstore list -l",
                                      "sstore_list"),
    'sstore_stats'    : ('local_exe', "/usr/sbin/isi_sstore stats",
                                      "sstore_stats"),
    'top'             : ('exe', "/usr/bin/top -S -b 1000 -d1", "top"),
    'uname'           : ('exe', "/usr/bin/uname -a", "uname"),
    'uptime'          : ('exe', '/usr/bin/uptime', 'uptime'),
    'varlog_all'      : ('tar', "cd /var", "varlog.tar log"),
    'varlog_recent'   : ('tar', "cd /var",
                         "varlog.tar --exclude '*.gz' --exclude '*.bz2' log"),
    'isi_iscan_report': ('local_tar', "cd /ifs/.ifsvar/idi", "iscan.tar tjob.*"),
    # from /var/crash
    'varcrash_ls'     : ('exe', "${SUDO}ls -lhR /var/crash", "varcrash_ls"),
    'isi_hangdump'    : ('tar', "cd /var/crash",
                         "isi_hangdump.tar isi_hangdump*"),
    'cores'           : ('tar', "cd /var/crash",
                         "cores.tar *.core.gz"),
    # transplanted from isi_hw_bug_info
    'camcontrol'      : ('exe', "${SUDO}/sbin/camcontrol devlist", 'camcontrol'),
    'kern.cam'        : ('exe', "/sbin/sysctl kern.cam", "kern.cam"),
    'hw.mpt0'         : ('exe', "/sbin/sysctl hw.mpt0", "hw.mpt0"),
    'dev.mps'         : ('exe', "/sbin/sysctl dev.mps", "dev.mps"),
    'isi.idi'         : ('exe', "/sbin/sysctl isi.idi", "isi.idi"),
    'isi.verify'      : ('exe', "/sbin/sysctl isi.verify", "isi.verify"),
    'isi_checkjournal': ('exe', "${SUDO}/usr/sbin/isi_checkjournal -i",
                                "isi_checkjournal"),
    # NOTE potentially unsafe, commented out until resolution of bug 43316.
    #'isi_mptutil'     : ('exe', "isi_mptutil 0 -cmv", 'isi_mptutil'),
    'bbu_log'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_bbu_util',
                      'bbu_log'),
    'bmc_logs' : ('tar',
                      ("${SUDO}/usr/bin/isi_hwtools/isi_bmc_logs gather && "
                       "cd /var/crash"),
                      "bmc_debug_logs.tar bmc_debug_logs.tar.bz2"),
    'bmc_sensor_errors' : ('exe',
                      "${SUDO}/usr/bin/isi_hwtools/isi_bmc_logs sensor",
                      "bmc_sensor_errors"),
    'bmc_sel2' : ('tar',
                      (("${SUDO}/sbin/mount /dev/mirror/hw /hw && cd /",
                      "${SUDO}/sbin/umount /hw")),
                      "sel.tar /hw/isilon_sel/sel*"),
    'bmc_sel'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -S -V -p g -i 0',
                      'bmc_sel'),
    'bmc_sel_ipmidashboard' : ('exe',
                               '${SUDO}/usr/bin/isi_hwtools/isi_bmc_logs sel',
                               'bmc_sel_ipmidashboard'),
    'bmc_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a bmc',
                      'bmc_info'),
    'car_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a car',
                      'car_info'),
    'cmc_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a cmc',
                      'cmc_info'),
    'fpc_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a fpc',
                      'fpc_info'),
    'car_bootloader_fw' : ('exe',
                           '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -a car -R 2EC0810d',
                           'car_bootloader_fw'),
    'cmc_bootloader_fw' : ('exe',
                           '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -a cmc -R 2EC0810d',
                           'cmc_bootloader_fw'),
    'cff_ps1_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -F -V -p D -a cmc -i 1',
                      'cff_ps1_info'),
    'cff_ps2_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -F -V -p D -a cmc -i 2',
                      'cff_ps2_info'),
    'ipmi_lan_settings' : ('exe',
                           '${SUDO}/usr/bin/ipmiutil lan print',
                           'ipmi_lan_settings'),
    'ipmi_cache_settings' : ('exe',
                             '${SUDO}/usr/bin/hexdump -C /etc/ipmi_command_cache_settings',
                             'ipmi_cache_settings'),
    'ipmi_reset_settings' : ('exe',
                             '${SUDO}/usr/bin/isi_gconfig -t hwsupport coldreset',
                             'ipmi_reset_settings'),
    'ipmi_hwmon_settings' : ('exe',
                             '${SUDO}/usr/bin/isi_gconfig -t hwsupport hwmon',
                             'ipmi_hwmon_settings'),
    'hw.ipmi' : ('exe', '/sbin/sysctl hw.ipmi', 'hw.ipmi'),
    # Infinity specific hwmon flags
    'hwmon_bbu' : ('exe', '/usr/bin/isi_hwmon -b', 'hwmon_bbu'),
    'hwmon_ecc' : ('exe', '/usr/bin/isi_hwmon -e', 'hwmon_ecc'),
    'hwmon_fsr' : ('exe', '/usr/bin/isi_hwmon -f', 'hwmon_fsr'),
    'hwmon_mca' : ('exe', '/usr/bin/isi_hwmon -c', 'hwmon_mca'),
    'hwmon_psu' : ('exe', '/usr/bin/isi_hwmon -r', 'hwmon_psu'),
    # Legacy hwmon flags
    'hwmon_ecc_logs' : ('exe', '/usr/bin/isi_hwmon -l', 'hwmon_ecc_logs'),
    'cdesexputil-chas0-zone0-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -p all chas0 zone0',
                      'cdesexputil-chas0-zone0-phy'),
    'cdesexputil-chas0-zone0-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -t all chas0 zone0',
                      'cdesexputil-chas0-zone0-topology'),
    'cdesexputil-chas0-zone1-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -p all chas0 zone1',
                      'cdesexputil-chas0-zone1-phy'),
    'cdesexputil-chas0-zone1-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -t all chas0 zone1',
                      'cdesexputil-chas0-zone1-topology'),
    'cdesexputil-fw'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -q',
                      'cdesexputil-fw'),
    'cdesexputil-debug'  : ('exe',
                      "${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -d ''",
                      'cdesexputil-debug'),
    'cdesexputil-ses'  : ('exe',
                      "${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -S",
                      'cdesexputil-ses'),
    'dsp_config_versions' : ('exe',
                      '/usr/bin/isi_hwtools/isi_dsptool dump_versions',
                      'dsp_config_versions'),
    'peer_serial_number' : ('exe',
                      'python -c "import isi.hw.hal; print isi.hw.hal.getPeerSerialNumber()"',
                      'peer_serial_number'),
    'isi_mpt_fw_status' : ('exe', '/usr/bin/isi_hwtools/isi_mpt_fw_status '\
                                  '2>/dev/null', 'isi_mpt_fw_status'),
    'isi_mpsutil_fw' : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_mpsutil -q",
                         "isi_mpsutil_fw"),
    'isi_mpsutil_ncq' : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_mpsutil -n",
                         "isi_mpsutil_ncq"),
    'isi_pmp_r' : ('exe', "/usr/bin/isi_pmp -r", "isi_pmp_r"),
    'isi_pmp_g' : ('exe', "/usr/bin/isi_pmp -g", "isi_pmp_g"),
    'isi_radish'      : ('exe', "${SUDO}/usr/bin/isi_radish -a", "isi_radish"),
    'isi_radish_bootdisk' : ('exe',
                         "${SUDO}/usr/bin/isi_radish -a 'Internal J3' 'Internal J4' 'Carrier board J3' 'Carrier board J4'",
                         "isi_radish_bootdisk"),
    'isi_smbios'      : ('exe', "${SUDO}/usr/bin/isi_smbios", "isi_smbios"),
    'isi_spcvutil'    : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_spcvutil info", "isi_spcvutil"),
    'lsiexputil-chas0-expn0-fw'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -q chas0 expn0',
                      'lsiexputil-chas0-expn0-fw'),
    'lsiexputil-chas0-expn0-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -p all chas0 expn0',
                      'lsiexputil-chas0-expn0-phy'),
    'lsiexputil-chas0-expn0-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -t all chas0 expn0',
                      'lsiexputil-chas0-expn0-topology'),
    'lsiexputil-chas0-expn0-log'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -l chas0 expn0',
                      'lsiexputil-chas0-expn0-log'),
    'lsiexputil-chas0-expn1-fw'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -q chas0 expn1',
                      'lsiexputil-chas0-expn1-fw'),
    'lsiexputil-chas0-expn1-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -p all chas0 expn1',
                      'lsiexputil-chas0-expn1-phy'),
    'lsiexputil-chas0-expn1-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -t all chas0 expn1',
                      'lsiexputil-chas0-expn1-topology'),
    'lsiexputil-chas0-expn1-log'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -l chas0 expn1',
                      'lsiexputil-chas0-expn1-log'),
    'isi_fcb_vpd_dump'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_fcb_vpd_tool dump',
                      'isi_fcb_vpd_dump'),
    'isi_fcb_vpd_dumpall'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_fcb_vpd_tool --devid all dump',
                      'isi_fcb_vpd_dumpall'),
    'isi_fcb_vpd_fdump'  : ('exe', '${SUDO}/usr/bin/isi_hwtools/isi_fcb_vpd_tool fdump',
                            'isi_fcb_vpd_fdump'),
    'gmirror_list'    : ('exe', '/sbin/gmirror list',
                               'gmirror_list'),
    'gmirror_status'  : ('exe', '/sbin/gmirror status',
                               'gmirror_status'),
    'gpart_show'  : ('exe', '/sbin/gpart show', 'gpart_show'),
    'mount' : ('exe', '/sbin/mount', 'mount'),
    'isi_var_migrate_status' : ('exe', '/usr/bin/isi_var_migrate --status',
                                'isi_var_migrate_status'),
    'isi_lox_info'  : ('exe', '/usr/bin/isi_lox_info -d', 'isi_lox_info'),
    'isi_bootdisk_status'  : ('exe', '${SUDO}/usr/bin/isi_bootdisk_status',
                               'isi_bootdisk_status'),
    'pciconf'      : ('exe', "${SUDO}/usr/sbin/pciconf -lbcve", "pciconf"),
    'vitutil-chas0-fw'  : ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -q chas0',
                            'vitutil-chas0-fw'),
    'vitutil-chas0-conf': ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -cvm chas0',
                            'vitutil-chas0-conf'),
    'vitutil-chas1-conf': ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -cvm chas1',
                            'vitutil-chas1-conf'),
    'vitutil-chas1-fw'  : ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -q chas1',
                            'vitutil-chas1-fw'),
    'isi_fputil'      : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_fputil -q",
                         "isi_fputil"),
    'dev.isp'         : ('exe', "/sbin/sysctl dev.isp", "dev.isp"),
    'isi_fc'          : ('local_exe', "${SUDO}/usr/bin/isi fc settings list", "isi_fc"),
    'isi_tape'        : ('local_exe', "${SUDO}/usr/bin/isi tape list -v", "isi_tape"),
    'efs.bam.dump_last_write_plan':
                        ('exe', "/sbin/sysctl efs.bam.dump_last_write_plan",
                                    'efs.bam.dump_last_write_plan'),
    'minidumps'       : ('tar', "cd /var/crash", "minidumps.tar vmcore*.*"),
    'lastgather_timestamp': ('local_exe', \
                             'ls -lT /ifs/data/Isilon_Support/*gather '\
                             ' /var/crash/support/*gather ' , \
                             'lastgather_timestamp'),
    'isi_statistics_pstat': ('local_exe', "/usr/bin/isi statistics pstat",
                                    "isi_statistics_pstat"),

    # all nodes
    'arp':
        ('exe',       "/usr/sbin/arp -an", "arp"),
    'df_all':
        ('exe',       "/bin/df -hi", "df_all"),
    'ibnetdiscover':
        ('exe',       "(/usr/bin/ibnetdiscover -P 1 -t 10; echo; " \
                      "/usr/bin/ibnetdiscover -P 2 -t 10)", "ibnetdiscover"),
    'ibstat':
        ('exe',       "/usr/bin/ibstat", "ibstat"),
    'isi_dump_fabric':
        ('exe',       "(/usr/bin/isi_dump_fabric int-a; echo; " \
                      "/usr/bin/isi_dump_fabric int-b)", "isi_dump_fabric"),
    'isi_auth_nodes':
        ('exe', "${SUDO}/usr/bin/isi_gather_auth_info -n", "isi_auth"),
    'isi_client_stats':
        ('exe',       "/usr/bin/isi_client_stats --notop -r 10",
            "isi_client_stats"),
    'iostat':
        ('exe',       "/usr/sbin/iostat -d -n 64", "iostat"),
    'isi_flexprotect':
        ('exe',       "/usr/bin/isi flexprotect", "isi_flexprotect"),
    'isi_hw_status_all':
        ('exe',      "${SUDO}/usr/bin/isi_hw_status -A", "isi_hw_status_all"),
    'isi_services':
        ('exe',      "${SUDO}/usr/bin/isi services -a", "isi_services"),
    'isi_testjournal':
        ('exe', "/usr/sbin/isi_testjournal", "isi_testjournal"),
    'ndp':
        ('exe', "/usr/sbin/ndp -an", "ndp"),
    'netstat_listen_queue':
        ('exe',       "/usr/bin/netstat -nL", "netstat_listen_queue"),
    'netstat_routes':
        ('exe',       "/usr/bin/netstat -anr", "netstat_routes"),
    'netstat_sockets':
        ('exe',       "/usr/bin/netstat -an", "netstat_sockets"),
    'ps':
        ('exe',       "/bin/ps auxww", "ps"),
    'ps_full':
        ('exe',       "/bin/ps auxwwHl", "ps_full"),
    'resolv.conf':
        ('exe',       "/bin/cat /etc/resolv.conf", "resolv.conf"),
    'hardening_report.rpt':
        ('local_exe',       "if [ -e /ifs/.ifsvar/CHE/reports/hardening_report.rpt ]; then /bin/cat /ifs/.ifsvar/CHE/reports/hardening_report.rpt; else echo 'No Hardening Errors to report'; fi", "hardening_report.rpt"),
    'hardening_engine.log':
        ('local_exe',       "/bin/cat /ifs/.ifsvar/CHE/log/hardening_engine.log", "hardening_engine.log"),
    'passwd':
        ('exe',       "/bin/cat /etc/passwd", "passwd"),

    # single node only
    'ib_subnet_master_local':
        ('local_exe', "/usr/bin/grep ': MA :' /var/log/opensm*.log",
                                "ib_subnet_master"),
    'isi_auth':
        ('local_exe', "${SUDO}/usr/bin/isi_gather_auth_info", "isi_auth"),
    # no --timeout as this is really an isi_classic command
    'isi_auth_mapping_list':
        ('local_exe', "${SUDO}/usr/bin/isi auth mapping list",
            "isi_auth_mapping_list"),
    'isi_auth_roles_list':
        ('local_exe',
            "${SUDO}/usr/bin/isi --timeout=15 auth roles list --verbose",
            "isi_auth_roles_list"),
    'isi_event_events_local':
        ('local_exe', "/usr/bin/isi event events list", "isi_event_events"),
    'isi_event_groups':
        ('local_exe', "/usr/bin/isi event groups list", "isi_event_groups"),
    'isi_flexprotect_local':
        ('local_exe', "/usr/bin/isi flexprotect", "isi_flexprotect"),
    'isi_ib_bug_info_local':
        ('local_exe', "/usr/bin/isi_ib_bug_info", "isi_ib_bug_info"),
    'isi_nfs_exports_local' :
        ('local_exe', "/usr/bin/isi_gather_nfs_exports",
            "isi_nfs_exports"),
    'isi_statistics':
        ('local_exe', "/usr/bin/isi statistics system --degraded --nodes all",
             "isi_statistics"),
    'isi_statistics_lsass':
        ('local_exe', "/usr/bin/isi --timeout 5 statistics protocol --degraded --long --protocols=lsass_in,lsass_out",
             "isi_statistics_lsass"),
    'isi_statistics_nfs':
        ('local_exe', "/usr/bin/isi --timeout 5 statistics protocol --degraded --long --protocols=nfs3,nfs4",
             "isi_statistics_nfs"),
    'isi_statistics_smb':
        ('local_exe', "/usr/bin/isi --timeout 5 statistics protocol --degraded --long --protocols=smb1,smb2",
             "isi_statistics_smb"),
    'isi_sync_policy_local':
        ('local_exe', "/usr/bin/isi sync pol ls -v", "isi_sync_policy"),
    'isi_sync_job_local':
        ('local_exe', "/usr/bin/isi sync job ls", "isi_sync_job"),
    'isi_radish_local':
        ('local_exe', "/usr/bin/isi_radish -a", "isi_radish"),
    'isi_job_status':
        ('local_exe', "/usr/bin/isi job status --verbose", "isi_job_status"),
    'isi_job_history':
        ('local_exe', "/usr/bin/isi job reports list", "isi_job_history"),
    'isi_job_list':
        ('local_exe', "/usr/bin/isi job jobs list", "isi_job_list"),
    'isi_job_schedule':
        ('local_exe', "/usr/bin/isi job types list --format list -v --all", "isi_job_schedule"),
    'isi_job_policy':
        ('local_exe', "${SUDO}/usr/bin/isi job policies list", "isi_job_policy"),
    'isi_network':
        ('local_exe', "${SUDO}/usr/bin/isi_gather_network_info",
            "isi_network"),
    'isi_network_interfaces_local':
        ('local_exe', "/usr/bin/isi network interfaces list -v",
                "isi_network_interfaces"),
    'isi_network_rules_local':
        ('local_exe', "/usr/bin/isi network rules list -v", "isi_network_rules"),
    'isi_network_pools_local':
        ('local_exe', "/usr/bin/isi network pools list -v",
                "isi_network_pools"),
    'isi_network_subnets_local':
        ('local_exe', "/usr/bin/isi network subnets list -v",
                "isi_network_subnets"),
    'isi_network_groupnets_local':
        ('local_exe', "/usr/bin/isi network groupnets list -v",
                "isi_network_groupnets"),
    'isi_network_dnscache_local':
        ('local_exe', "/usr/bin/isi network dnscache view",
                "isi_network_dnscache"),
    'isi_network_external_local':
        ('local_exe', "/usr/bin/isi network external view",
                "isi_network_external"),
    'isi_services_local':
        ('local_exe', "/usr/bin/isi services -a", "isi_services"),
    'isi_status_local':
        ('local_exe', "/usr/bin/isi status", "isi_status"),
    'isi_tape_local':
        ('local_exe', "/usr/bin/isi tape list -v", "isi_tape"),
    # Grab pstore, but remove their current AD password for customer security
    'pstore_dump':
        ('local_exe', "grep -v 'Password.value' /ifs/.ifsvar/pstore.gc",
            "pstore_dump"),
    'pstore_changes_dump':
        ('local_exe',
            "grep -v 'Password.value' /ifs/.ifsvar/pstore_changes.log",
            "pstore_changes_dump"),
    'snapshot_list_local':
        ('local_exe', "/usr/bin/isi snap list", "snapshot_list"),
    'snapshot_schedule_local':
        ('local_exe', "/usr/bin/isi snap schedule list", "snapshot_schedule"),
    'snapshot_settings_local':
        ('local_exe', "/usr/bin/isi snap settings view", "snapshot_settings"),
    'snapshot_usage_local':
        ('local_exe', "/usr/bin/isi snap snap list -v --format table --state all",
            "snapshot_usage"),
    'uptime_local':
        ('local_exe', '/usr/bin/uptime', 'uptime'),
    'varlog_messages':
        ('tar', "cd /var/log", "messages.tar messages"),
    'isi_vol_copy':
        ('local_tar', "cd /ifs/.ifsvar/modules/isi_vol_copy", "isi_vol_copy.tar"),
    'etc_newsyslog':
        ('tar', "cd /etc", "newsyslog_conf.tar newsyslog*"),
    #added for MAV 7.0
    'efs.bam.ec.stats' :
        ('exe', "/sbin/sysctl efs.bam.ec.stats", 'efs.bam.ec.stats'),
    #added for MAV 7.0
    'efs.bam.ec.known_logstores' :
            ('exe', "/sbin/sysctl efs.bam.ec.known_logstores", 'efs.bam.ec.known_logstores'),
    'audit_topics':
        ('local_exe', "/usr/bin/isi audit topics list", "isi_audit_topics"),
    'smartlock_domain_long':
        ('local_exe', "/usr/bin/isi worm domains list -v", "isi_worm_domains_long"),
    'smartlock_domain_wide':
        ('local_exe', "/usr/bin/isi worm domains list", "isi_worm_domains_wide"),
    'efs.lbm' :
        ('exe', "/sbin/sysctl efs.lbm", 'efs.lbm'),
    'efs.bam.bsw' :
        ('exe', "/sbin/sysctl efs.bam.bsw", 'efs.bam.bsw'),
    'isi_filepool_policies':
        ('local_exe', "/usr/bin/isi filepool policies list -v", "isi_filepool_policies"),
    'efs.bam.layout.ssd.sys_btree.mirrors' :
        ('exe', "/sbin/sysctl efs.bam.layout.ssd.sys_btree.mirrors", 'efs.bam.layout.ssd.sys_btree.mirrors'),
    'isi_filepool_default_policies':
        ('local_exe', "/usr/bin/isi filepool default-policy view", "isi_filepool_default_policies"),
    'efs.gmp.current_info': ('exe', '/sbin/sysctl efs.gmp.current_info', 'efs.gmp.current_info'),
    'dmidecode_memory':
        ('exe', "${SUDO}/usr/local/sbin/dmidecode -t memory", "dmidecode_memory"),
    'dmesg_boot':
        ('tar', "cd /var/run", "dmesg.tar dmesg.boot*"),
    'isi_gconfig': ('local_exe', "${SUDO}/usr/bin/isi_gconfig", "isi_gconfig"),
    'isi_job_gconfig':
        ('local_exe', "${SUDO}/usr/bin/isi_gconfig -t job-config", "isi_job_gconfig"),
    'cache_stats':
        ('exe', "${SUDO}/usr/bin/isi_cache_stats -v", "cache_stats"),
    'isi_event_alerts':
        ('local_exe', "${SUDO}/usr/bin/isi event alerts list -v", "isi_event_alerts"),
    'isi_event_channel':
        ('local_exe', "${SUDO}/usr/bin/isi event channel list -v", "isi_event_channel"),
    'isi_event_settings':
        ('local_exe', "${SUDO}/usr/bin/isi event settings view", "isi_event_settings"),
    'efs.journal.indeterminate_txns' :
        ('exe', "/sbin/sysctl efs.journal.indeterminate_txns", 'efs.journal.indeterminate_txns'),
    'efs.journal.journal_nsb_data' :
        ('exe', "/sbin/sysctl efs.journal.journal_nsb_data",
                'efs.journal.journal_nsb_data'),
    'efs.journal.mirror_state' :
        ('exe', "/sbin/sysctl efs.journal.mirror_state",
                'efs.journal.mirror_state'),
    'efs.journal.stats.indeterminate' :
        ('exe', "/sbin/sysctl efs.journal.stats.indeterminate", 'efs.journal.stats.indeterminate'),
    'efs.journal.transactions' :
        ('exe', "/sbin/sysctl efs.journal.transactions", 'efs.journal.transactions'),
    'efs.acct'                   : ('exe', '/sbin/sysctl -N efs.acct | ' \
                                    '/usr/bin/grep -v "efs.acct.raas" | ' \
                                    '/usr/bin/grep -v "efs.acct.actors" | ' \
                                    '/usr/bin/xargs /sbin/sysctl',
                                    'efs.acct'),
    'isi_statistics_workload'    : ('exe', 'isi statistics workload --limit 100',
                                    'isi_statistics_workload'),
    'isi_storagepool_settings':
        ('local_exe', "${SUDO}/usr/bin/isi storagepool settings view", "isi_storagepool_settings"),
    'isi_storagepool_nodepool':
        ('local_exe', "${SUDO}/usr/bin/isi storagepool nodepool list -v", "isi_storagepool_nodepool"),
    'isi_hdfs_settings_view':
        ('local_exe', "/usr/bin/isi hdfs settings view", "isi_hdfs_settings_view"),
    'isi_hdfs_proxyusers_list':
        ('local_exe', "/usr/bin/isi hdfs proxyusers list -v", "isi_hdfs_proxyusers_list"),
    'isi_gconfig_hdfs':
        ('local_exe', "${SUDO}/usr/bin/isi_gconfig -t hdfs", "isi_gconfig_hdfs"),
    'isi_hdfs_racks':
        ('local_exe', "/usr/bin/isi hdfs racks list --verbose", "isi_hdfs_racks"),
    'isi_hdfs_log_level':
        ('local_exe', "/usr/bin/isi hdfs log-level view", "isi_hdfs_log_level"),
    'isi_hdfs_zone_settings':
          ('local_exe', "/usr/bin/isi_gather_hdfs_zone_settings", "isi_hdfs_zone_settings"),
    'firmware_ls':
        ('local_exe', "${SUDO}ls -lhR /ifs/.ifsvar/modules/hardware/drives/firmware", "firmware_ls"),
    'isi_audit_global_settings':
        ('local_exe', "/usr/bin/isi audit settings global view", "isi_audit_global_settings"),
    'isi_audit_zone_settings':
        ('local_exe', "/usr/bin/isi_gather_audit_zone_settings", "isi_audit_zone_settings"),
    'isi_cloud_pools_settings':
        ('local_exe', "${SUDO}/usr/bin/isi cloud pools list -v", "isi_cloud_pool_settings"),
    'isi_cloud_accounts_settings':
        ('local_exe', "${SUDO}/usr/bin/isi cloud accounts list -v", "isi_cloud_accounts_settings"),
    'isi_cloud_access_settings':
        ('local_exe', "${SUDO}/usr/bin/isi cloud access list -v", "isi_cloud_access_settings"),
    'isi_cloud_jobs_settings':
        ('local_exe', "${SUDO}/usr/bin/isi cloud jobs list -v", "isi_cloud_jobs_settings"),
}

# Command line option can replace 'varlog_all' with 'varlog_recent'
EXEMPT = [
    'varlog_all',    # always must be first

    'abr_config',
    'arp',
    'aspera_config',
    'camcontrol',
    'celog_node_db',
    'celog_ifsvar_db',
    'configs',
    'isi_phone_home_config_local',
    'isi_phone_home_config_node',
    'cto_summary',
    'dedupe_reports',
    'dedupe_settings',
    'dedupe_stats',
    'sfm.shadow.stats',
    'sfm.clone.stats',
    'sfm.dedupe.stats',
    'dev.isp',
    'dev.ioat',
    'dev.ntb_hw.0',
    'compliance',
    'df',
    'du',
    'efs.acct',
    'efs.bam.dump_last_write_plan',
    'efs.bam.ec.stats',
    'efs.bam.bsw',
    'efs.bam.ec.known_logstores',
    'efs.gmp.group',
    'efs.l3',
    'efs.lbm.drive_space',
    'efs.lbm',
    'efs.snapshot',
    'efs.stf',
    'etcifs',
    'etc',
    'fstat',
    'gconfig_local',
    'gconfig_ifsvar',
    'kern.cam.ada',
    'hw.disk_event',
    'hw.ecclist',
    'hw.iosched',
    'hw.mnv0',
    'hw.mnvi',
    'hw.mpt0',
    'dev.mps',
    'ibnetdiscover',
    'ibstat',
    'isi_dump_fabric',
    'ifconfig',
    'ifsvar_db_pkg',
    'ifsvar_db_pkg_tars',
    'ifsvar_etc',
    'ifsvar_modules',
    'ifsvar_tmp',
    'ifsvar_smb',
    'ifsvar_modules_jobengine_cp',
    'ifsvar_modules_tardis',
    'ifsvar_upgrade',
    'isi.idi',
    'isi.verify',
    'isi_iscan_report',
    'isi_auth',
    'isi_auth_nodes',
    'isi_auth_mapping_list',
    'isi_auth_roles_list',
    'isi_checkjournal',
    'isi_devices_drive_list',
    'isi_devices_firmware_list',
    'isi_devices_firmware_summary',
    'isi_drivenum',
    'isi_disk_inject',
    'isi_diskpool',
    'efs.bam.disk_pool_db',
    'isi_event_events',
    'isi_event_groups',
    'isi_dmi_log',
    'isi_fc',
    'isi_gconfig',
    'isi_hangdump',
    'cores',
    'isi_hw_status',
    'isi_ib_bug_info',
    'isi_job_status',
    'isi_job_history',
    'isi_job_list',
    'isi_job_schedule',
    'isi_job_policy',
    'isi_license',
    'isi_license.xml',
    'isi_esrs',
    'isi_remote_support',
    'isi_network',
    'isi_network_interfaces_local',
    'isi_network_rules_local',
    'isi_network_pools_local',
    'isi_network_subnets_local',
    "isi_network_groupnets_local",
    "isi_network_dnscache_local",
    "isi_network_external_local",
    'isi_nfs_exports',
    'isi_nfs_aliases',
    'isi_psi_tool_receipt',
    'isi_radish',
    'isi_radish_bootdisk',
    'isi_services_local',
    'isi_spcvutil',
    'isi_smb',
    'isi_smbios',
    'isi_stat',
    'isi_statistics',
    'isi_statistics_drive',
    'isi_statistics_heat',
    'isi_statistics_lsass',
    'isi_statistics_nfs',
    'isi_statistics_smb',
    'isi_statistics_workload',
    'isi_status',
    'isi_upgrade_archive_gather',
    'isi_upgrade_view',
    'isi_upgrade_list',
    'isi_stat-p',
    'isi_sync_policy',
    'isi_tape',
    'isi_ufp',
    'isi_usysctl_psi',
    'kern.cam',
    'kern.geom',
    'kern.proc.all_stacks',
    'kern.malloc_pigs',
    'kern.mbaudit',
    'kern.openfiles',
    'licensing_ifsvar',
    'lastgather_timestamp',
    'minidumps',
    'ndp',
    'netstat',
    'isi_nfs4mgmt',
    'nfsstat',
    'node_info_cache',
    'override',
    'patch_list',
    'patch_list_local',
    'patch_installed_db',
    'patch_activity_db',
    'patch_cluster_db',
    'ps',
    'ps_full',
    'pstore_changes_dump',
    'pstore_dump',
    'rpcinfo',
    'resolv.conf',
    'hardening_engine.log',
    'hardening_report.rpt',
    'passwd',
    'quotas',
    'quotas_verbose',
    'sockstat',
    'snapshot_list_local',
    'snapshot_schedule_local',
    'snapshot_settings_local',
    'snapshot_usage_local',
    'sin_cache',
    'sstore_stats',
    'top',
    'uname',
    'upgrade_agent_persistent_db',
    'upgrade_local',
    'uptime',
    'varcrash_ls',
    'isi_mpt_fw_status',
    'bbu_log',
    'bmc_logs',
    'bmc_sel',
    'bmc_sel2',
    'bmc_sel_ipmidashboard',
    'bmc_sensor_errors',
    'bmc_info',
    'car_info',
    'cmc_info',
    'car_bootloader_fw',
    'cmc_bootloader_fw',
    'cff_ps1_info',
    'cff_ps2_info',
    'fpc_info',
    'ipmi_lan_settings',
    'ipmi_cache_settings',
    'ipmi_reset_settings',
    'ipmi_hwmon_settings',
    'hw.ipmi',
    'hwmon_bbu',
    'hwmon_ecc',
    'hwmon_ecc_logs',
    'hwmon_fsr',
    'hwmon_mca',
    'hwmon_psu',
    'cdesexputil-chas0-zone0-phy',
    'cdesexputil-chas0-zone0-topology',
    'cdesexputil-chas0-zone1-phy',
    'cdesexputil-chas0-zone1-topology',
    'cdesexputil-fw',
    'cdesexputil-debug',
    'cdesexputil-ses',
    'peer_serial_number',
    'dsp_config_versions',
    'isi_mpsutil_fw',
    'isi_mpsutil_ncq',
    'isi_pmp_r',
    'isi_pmp_g',
    'lsiexputil-chas0-expn0-fw',
    'lsiexputil-chas0-expn0-phy',
    'lsiexputil-chas0-expn0-log',
    'lsiexputil-chas0-expn0-topology',
    'lsiexputil-chas0-expn1-fw',
    'lsiexputil-chas0-expn1-phy',
    'lsiexputil-chas0-expn1-log',
    'lsiexputil-chas0-expn1-topology',
    'isi_fcb_vpd_dump',
    'isi_fcb_vpd_dumpall',
    'isi_fcb_vpd_fdump',
    'gmirror_list',
    'gmirror_status',
    'gpart_show',
    'mount',
    'isi_var_migrate_status',
    'isi_bootdisk_status',
    'pciconf',
    'vfs.nfs',
    'vfs.nlm',
    'vitutil-chas0-conf',
    'vitutil-chas0-fw',
    'vitutil-chas1-conf',
    'vitutil-chas1-fw',
    'isi_fputil',
    'vm.uma_zalloc_pigs',
    'journal_stats',
    'etc_newsyslog',
    'isi_lox_info',
    'isi_audit_global_settings',
    'audit_topics',
    'smartlock_domain_long',
    'smartlock_domain_wide',
    'isi_statistics_pstat',
    'zsh_history',
    'isi_filepool_policies',
    'isi_filepool_default_policies',
    'efs.bam.layout.ssd.sys_btree.mirrors',
    'efs.gmp.current_info',
    'zsh_history_compadmin',
    'dmidecode_memory',
    'isi_job_gconfig',
    'cache_stats',
    'isi_event_alerts',
    'isi_event_channel',
    'isi_event_settings',
    'efs.journal.indeterminate_txns',
    'efs.journal.journal_nsb_data',
    'efs.journal.mirror_state',
    'efs.journal.stats.indeterminate',
    'efs.journal.transactions',
    'isi_storagepool_settings',
    'isi_storagepool_nodepool',
    'isi_hdfs_settings_view',
    'isi_hdfs_proxyusers_list',
    'isi_gconfig_hdfs',
    'isi_hdfs_racks',
    'isi_hdfs_log_level',
    'isi_hdfs_zone_settings',
    'firmware_ls',
    'isi_audit_zone_settings',
]

# GROUPS holds the groups of utilities that can be specified by the
# --group command line option.
GROUPS = {
    "abr": [
        "abr_config",
        "cto_summary",
    ],
    "cluster": [
        "isi_status",
        "isi_upgrade_view",
        "isi_upgrade_list",
        "isi_upgrade_archive_gather",
        "isi_event_events_local",
        "isi_services_local",
        "isi_status_local",
        "isi_license",
        "isi_license.xml",
        "isi_esrs",
        "isi_remote_support",
        "hardening_report.rpt",
        "hardening_engine.log",
        "licensing_ifsvar",
        "isi_tape_local",
        "patch_list",
        "patch_cluster_db",
        "uname",
        #"isi_firmware",     # unsafe, unimplemented; bug 43316
    ],
    "fs": [
        "isi_checkjournal",
        "isi_testjournal",
        "efs.gmp.group",
        "isi_job_status",
        "isi_job_history",
        "isi_job_list",
        "isi_job_schedule",
        "isi_job_policy",
        "isi_flexprotect_local",
        "efs.journal.journal_nsb_data",
        "efs.journal.mirror_state",
    ],
    "auth": [
        "isi_auth",
        "isi_auth_nodes",
        "isi_auth_mapping_list",
        "isi_auth_roles_list",
        "pstore_changes_dump",
        "pstore_dump",
        "isi_statistics_lsass",
        "passwd",
    ],
    "cloudpools": [
        "isi_cloud_pools_settings",
        "isi_cloud_accounts_settings",
        "isi_cloud_access_settings",
        "isi_cloud_jobs_settings",
    ],
    "protocol": [
        "isi_nfs_exports_local",
        "isi_smb",
        "isi_statistics_nfs",
        "isi_statistics_smb",
        "isi_nfs4mgmt",
        "nfsstat",
        "ps_full",
        "rpcinfo",
        "vfs.nfs",
        "vfs.nlm",
    ],
    "messages": [
        "varlog_messages",
    ],
    "bootmessages": [
        "dmesg_boot",
    ],
    "network": [
        "arp",
        "ibnetdiscover",
        "ibstat",
        "isi_dump_fabric",
        "ifconfig",
        "resolv.conf",
        "ndp",
        "netstat",
        "netstat_listen_queue",
        "netstat_sockets",
        "netstat_routes",
        "sockstat",
        "isi_client_stats",
        "isi_network_interfaces_local",
        "isi_network_rules_local",
        "isi_network_pools_local",
        "isi_network_subnets_local",
        "isi_network_groupnets_local",
        "isi_network_dnscache_local",
        "isi_network_external_local",
    ],
    "application": [
        "snapshot_list_local",
        "snapshot_usage_local",
        "snapshot_schedule_local",
        "snapshot_settings_local",
        "efs.snapshot",
        "efs.stf",
        "quotas_verbose",   # quota is local only command now
        "isi_sync_policy_local",
        "isi_sync_job_local",
    ],
    "ib": [
        "isi_ib_bug_info_local",
        "ib_subnet_master_local",
    ],
    "migration":[
         "isi_vol_copy",
    ],
    "node": [
        'isi_status',
        'isi_upgrade_view',
        'isi_upgrade_list',
        'bbu_log',
        'bmc_info',
        'bmc_logs',
        'bmc_sel',
        'bmc_sel2',
        'bmc_sel_ipmidashboard',
        'bmc_sensor_errors',
        'car_info',
        'cmc_info',
        'car_bootloader_fw',
        'cmc_bootloader_fw',
        'cff_ps1_info',
        'cff_ps2_info',
        'fpc_info',
        'ipmi_lan_settings',
        'ipmi_cache_settings',
        'ipmi_reset_settings',
        'ipmi_hwmon_settings',
        'hw.ipmi',
        'hwmon_bbu',
        'hwmon_ecc',
        'hwmon_ecc_logs',
        'hwmon_fsr',
        'hwmon_mca',
        'hwmon_psu',
        'peer_serial_number',
        'dsp_config_versions',
        "isi_devices_drive_list",
        "isi_devices_firmware_list",
        "isi_drivenum",
        'isi_fcb_vpd_dump',
        'isi_fcb_vpd_dumpall',
        'isi_fcb_vpd_fdump',
        "isi_hw_status_all",
        "isi_pmp_g",
        "isi_pmp_r",
        "isi_psi_tool_receipt",
        "isi_usysctl_psi",
        "camcontrol",
        "isi_dmi_log",
        "uptime_local",
        "isi_radish_local",
        "isi_radish_bootdisk",
        "isi_bootdisk_status",
        "gmirror_list",
        "gmirror_status",
        "gpart_show",
        'mount',
        "isi_var_migrate_status",
        "kern.cam.ada",
        "df_all",
        "varcrash_ls",
        "kern.geom",
        "kern.proc.all_stacks",
        "pciconf",
        "etc_newsyslog",
        "isi_lox_info",
        "cores",
    ],
    "storage": [
        "camcontrol",
        'cdesexputil-fw',
        'cdesexputil-chas0-zone0-phy',
        'cdesexputil-chas0-zone0-topology',
        'cdesexputil-chas0-zone1-phy',
        'cdesexputil-chas0-zone1-topology',
        "dev.ioat",
        "dev.ntb_hw.0",
        "kern.cam.ada",
        "hw.mpt0",
        "dev.mps",
        "hw.disk_event",
        "hw.ecclist",
        "hw.iosched",
        "hw.mnv0",
        "hw.mnvi",
        "kern.cam",
        "gmirror_list",
        "gmirror_status",
        "gpart_show",
        'mount',
        "isi_var_migrate_status",
        "isi_bootdisk_status",
        "isi_radish_bootdisk",
        "isi_devices_drive_list",
        "isi_devices_firmware_list",
        "isi_devices_firmware_summary",
        "isi_drivenum",
        "isi_radish",
        "isi_mpt_fw_status",
        "isi_mpsutil_fw",
        "isi_mpsutil_ncq",
        "lsiexputil-chas0-expn0-fw",
        "lsiexputil-chas0-expn0-phy",
        "lsiexputil-chas0-expn0-topology",
        "lsiexputil-chas0-expn0-log",
        "lsiexputil-chas0-expn1-fw",
        "lsiexputil-chas0-expn1-phy",
        "lsiexputil-chas0-expn1-topology",
        "lsiexputil-chas0-expn1-log",
        "vitutil-chas0-fw",
        "vitutil-chas0-conf",
        "vitutil-chas1-conf",
        "vitutil-chas1-fw",
    ],
    "usage": [
        "uptime",
        "top",
        "isi_statistics",
        "fstat",
        "kern.openfiles",
        "kern.mbaudit",
        "iostat",
    ],
    "logs": [
        "varlog_recent",
    ],
    "acct": [
        'efs.acct',
        'isi_statistics_workload',
    ],
    "hdfs": [
        "isi_hdfs_settings_view",
        "isi_hdfs_proxyusers_list",
        "isi_hdfs_racks",
        "isi_gconfig_hdfs",
        "isi_hdfs_log_level",
        "isi_hdfs_zone_settings",
    ],
}

# Commands for email body. node means executed on local node,
# cluster means cluster wide.
MAIL_COMMAND = [
        ('uname -a', 'node'),
        ('isi_hw_status -i', 'cluster'),
        ('isi status', 'node'),
        ('isi status -n', 'node'),
        ('isi event groups list', 'node'),
        ('isi_hw_status -A', 'cluster'),
        ('isi_bootdisk_status', 'cluster'),
        ('isi devices drive list', 'cluster'),
        ('date; uptime', 'cluster'),
        ('df', 'cluster'),
        ('isi_classic job status -v', 'node'),
        ('ps auxcjl', 'cluster'),
        ('ifconfig -a', 'cluster')
]


class Cluster: pass
class Options: pass

# Globals
cluster = Cluster()
options = Options()
log = ''    #used later on to write to gather-status log file
re_upload = ''

def is_ifs_writeable():
    global SUPPORT_DIR

    if not os.path.exists(SUPPORT_DIR):
        parentdir = os.path.dirname(SUPPORT_DIR.rstrip("/"))
        try:
            os.utime(parentdir, (-1, -1))
        except OSError, e:
            if e.errno == 30: # Read-only file system
                print "Read-only file system: %s. Attempting /var/crash." % SUPPORT_DIR
            else:
                # some error, still bail out
                print "Attempting /var/crash as %s has OS error: %s" % \
                    (parentdir, os.strerror(e.errno))

            return False
    else:
        try:
            os.utime(SUPPORT_DIR, (-1, -1))
        except OSError, e:
            if e.errno == 30: # Read-only file system
                print "Read-only file system: %s. Attempting /var/crash." % SUPPORT_DIR
            else:
                print "Attempting /var/crash as %s has OS error: %s" % \
                    (parentdir, os.strerror(e.errno))

            return False

    return True

def update_support_dirs(options):
    global SUPPORT_DIR
    global TMP_SUPPORT_DIR
    global PKG_SUPPORT_DIR
    global GATHER_SCRIPT
    global LOCAL_GATHER_SCRIPT
    global ISI_NODES_OUTPUT
    global ISI_FIRMWARE_OUTPUT
    global CELOG_EVENTS_INFO
    global PACKAGE_INFO
    global SAVED_COMMAND_LINE
    global LOCAL_TMP_SUPPORT_DIR
    global INCREMENTAL_FILE
    global FULLGATHER_FILE
    global GATHER_RUN_FILE
    global IFS_READONLY

    if options.tempdir:
        if options.tempdir[0] != '/':
            error("Only full path tempdir specifications allowed.")
        SUPPORT_DIR = options.tempdir
        if options.tempdir[0:4] != '/ifs':
            options.use_ifs = False
    elif options.use_ifs:
        SUPPORT_DIR = IFS_SUPPORT_DIR
        if not is_ifs_writeable():
            IFS_READONLY = True
            SUPPORT_DIR = VAR_SUPPORT_DIR
            options.use_ifs = False
    else:
        SUPPORT_DIR = VAR_SUPPORT_DIR

    # generate a unique random tempdir. this is not safe against races, but
    # shouldn't need to be.
    chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
    while True:
        tmpdir = ""
        for i in range(16):
            tmpdir += chars[ random.randint(0, len(chars)-1) ]
        # this directory is expected to end in "/" elsewhere in the code, but
        # to verify that a file doesn't exist by the name of the directory
        # we wish to create, we have to check if the directory exists without
        # the trailing slash, first.
        TMP_SUPPORT_DIR = os.path.join(SUPPORT_DIR, _TMP_SUPPORT_DIR, tmpdir)
        if not os.path.exists(TMP_SUPPORT_DIR):
            TMP_SUPPORT_DIR  += "/"
            break

    if options.tardir:
        PKG_SUPPORT_DIR   = options.tardir
    else:
        PKG_SUPPORT_DIR   = os.path.join(SUPPORT_DIR, _PKG_SUPPORT_DIR)


    INCREMENTAL_FILE      = os.path.join(SUPPORT_DIR, _INCREMENTAL_FILE)
    FULLGATHER_FILE       = os.path.join(SUPPORT_DIR, _FULLGATHER_FILE)
    GATHER_SCRIPT         = os.path.join(TMP_SUPPORT_DIR, _GATHER_SCRIPT)
    LOCAL_GATHER_SCRIPT   = os.path.join(TMP_SUPPORT_DIR, _LOCAL_GATHER_SCRIPT)
    ISI_NODES_OUTPUT      = os.path.join(TMP_SUPPORT_DIR, _ISI_NODES_OUTPUT)
    ISI_FIRMWARE_OUTPUT   = os.path.join(TMP_SUPPORT_DIR, _ISI_FIRMWARE_OUTPUT)
    CELOG_EVENTS_INFO     = os.path.join(TMP_SUPPORT_DIR, _CELOG_EVENTS_INFO)
    PACKAGE_INFO          = os.path.join(TMP_SUPPORT_DIR, _PACKAGE_INFO)
    SAVED_COMMAND_LINE    = os.path.join(TMP_SUPPORT_DIR, _SAVED_COMMAND_LINE)
    LOCAL_TMP_SUPPORT_DIR = os.path.join(TMP_SUPPORT_DIR, _LOCAL_TMP_SUPPORT_DIR)
    GATHER_RUN_FILE_NAME = 'gather-%d-%d.xml' % \
            (int(array.getLocalNode().lnn()), os.getpid())
    GATHER_RUN_FILE = os.path.join(RUN_SUPPORT_DIR, GATHER_RUN_FILE_NAME)

    # Make sure the dirs exist locally first
    makeDir(SUPPORT_DIR)
    makeDir(TMP_SUPPORT_DIR)
    makeDir(LOCAL_TMP_SUPPORT_DIR)
    makeDir(PKG_SUPPORT_DIR)

    # Let the main thread create benchmark directory
    # to avoid slaves racing for directory creation.
    if options.use_ifs:
        BENCHMARK_DIR = TMP_SUPPORT_DIR + "benchmark"
        makeDir(BENCHMARK_DIR)

    if options.symlinkdir:
        makeDir(options.symlinkdir)

def writeGatherRunFile():
    global TMP_SUPPORT_DIR
    global GATHER_RUN_FILE
    global GATHER_NODE_FD

    try:
        grfd = os.open(GATHER_RUN_FILE,
                os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0755)

    except (OSError, IOError), e:
        verbose('Unable to create run file on ifs. Attempting var.')
        makeDir(RUN_VAR_SUPPORT_DIR)
        GATHER_RUN_FILE = GATHER_RUN_FILE.replace(RUN_SUPPORT_DIR,
                    RUN_VAR_SUPPORT_DIR)
        try:
            grfd = os.open(GATHER_RUN_FILE,
                    os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0755)
        except (OSError, IOError), e:
            error('Could not create gather run file:' + str(e))

    grfile = os.fdopen(grfd, 'w')

    root = etree.Element('gather-run')
    tmpdir = etree.SubElement(root, 'tmpdir')
    tmpdir.text = TMP_SUPPORT_DIR
    et = etree.ElementTree(root)
    et.write(grfile)

    #TODO handle error!
    try:
        GATHER_NODE_FD = os.open(GATHER_NODE_FILE_PATH, os.O_EXLOCK | os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0755)
        local_node_lnn = str(array.getLocalNode().lnn())
        os.write(GATHER_NODE_FD, local_node_lnn.zfill(3))
    except Exception as e:
        #ignore errors
        print "Failed to create node ID file - check for other running gathers"

def cleanupGatherRunFile():
    try:
        os.unlink(GATHER_RUN_FILE)
    except Exception, e:
        verbose('Could not cleanup gather run file: ' + str(e))
    if GATHER_NODE_FD is not None:
        try:
            os.close(GATHER_NODE_FD)
            os.unlink(GATHER_NODE_FILE_PATH)
        except Exception, e:
            print e;
# this routine creates both the _gather script that is run by all nodes and
# the local_gather script that is only called once per cluster, on the
# invoking node.  run_utils should be a list of the names of utilities in
# the UTILITIES dict.
def generate_scripts(run_utils):
    # Determine if cluster is in compliance mode
    try:
        flags    = os.O_RDWR|os.O_CREAT|os.O_TRUNC
        header = "#!/usr/local/bin/bash\nHOST=`hostname`\n"
        if is_compliance():
            header += "SUDO='sudo '\n"
        footer = ""

        # timing logic
        header += "SCRIPT_START=`{ts_cmd}`\n".format(ts_cmd=TIMESTAMP_COMMAND)
        footer += ("SCRIPT_END=`{ts_cmd}`\n"
                   "echo SCRIPT_RUNTIME: "
                   "`echo \"$SCRIPT_END-$SCRIPT_START\" | bc`\n"
                   ).format(ts_cmd=TIMESTAMP_COMMAND)

        # script must exit cleanly
        footer += "exit 0\n"

        localfd  = os.open(LOCAL_GATHER_SCRIPT, flags, 0755)
        os.write(localfd, header)

        if not options.local_only:
            gatherfd = os.open(GATHER_SCRIPT, flags, 0755)
            os.write(gatherfd, header)

        for util in run_utils:
            exe,cmd,name = UTILITIES.get(util)
            cleanup = None
            if len(cmd) == 2 and isinstance(cmd, tuple):
                cmd, cleanup = cmd
            if exe == 'exe' and not options.local_only:
                line = "%s 1> %s$HOST/%s 2>&1" % \
                    (cmd, TMP_SUPPORT_DIR, name.lstrip())
            elif exe == 'tar' and not options.local_only:
                if options.incremental:
                    line = "%s && /usr/bin/tar -c --newer-mtime-than %s -f %s$HOST/%s 1>/dev/null 2>&1" % \
                        (cmd, INCREMENTAL_FILE, TMP_SUPPORT_DIR, name.lstrip())
                    # Remove the tar file if it's 0 bytes in length
                    TarfileName=name.split(' ')[0]
                    TarfileName= "%s$HOST/%s" % \
                        (TMP_SUPPORT_DIR, TarfileName)
                    line += "\n"
                    line += '[ -s "%s" ] || rm "%s"' % \
                        (TarfileName, TarfileName)
                else:
                    line = "%s && /usr/bin/tar cf %s$HOST/%s 1>/dev/null 2>&1" % \
                        (cmd, TMP_SUPPORT_DIR, name.lstrip())
            elif exe == 'local_tar' or (options.local_only and exe == 'tar'):
                if options.incremental:
                    line = "%s && /usr/bin/tar -c --newer-mtime-than %s -f %slocal/%s 1>/dev/null 2>&1" % \
                        (cmd, INCREMENTAL_FILE, TMP_SUPPORT_DIR, name.lstrip())
                    # Remove the tar file if it's 0 bytes in length
                    TarfileName=name.split(' ')[0]
                    TarfileName= "%slocal/%s" % (TMP_SUPPORT_DIR, TarfileName)
                    line += "\n"
                    line += '[ -s "%s" ] || rm "%s"' % \
                        (TarfileName, TarfileName)
                elif util == "isi_vol_copy":
                    dirs = ""

                    if not os.path.exists('/ifs/.ifsvar/modules/isi_vol_copy'):
                        continue

                    isi_vol_cmd = "isi_vol_copy -list -detail"
                    (exit, output) = procs.get_cmd_output(isi_vol_cmd)

                    if exit:
                        continue

                    if re.search("Failed",str(output)) == None:
                        continue

                    fvol = open(TMP_SUPPORT_DIR+"vol_copy_list.txt","w")
                    for idx,mig in enumerate(output):
                        if idx > 1:
                            mig = mig.lstrip()
                            dirs = str(mig.split(" ")[0])
                            fvol.write(dirs)
                            fvol.write("\n")
                    fvol.close()
                    line = "%s && /usr/bin/tar cfT %s$HOST/%s %s 1>/dev/null 2>&1" % \
                        (cmd, TMP_SUPPORT_DIR, name.lstrip(), TMP_SUPPORT_DIR+"vol_copy_list.txt")
                else:
                    line = "%s && /usr/bin/tar cf %slocal/%s 1>/dev/null 2>&1" % \
                        (cmd, TMP_SUPPORT_DIR, name.lstrip())
            elif exe == 'local_exe' or (options.local_only and exe == 'exe'):
                line = "%s 1> %slocal/%s 2>&1" % \
                    (cmd, TMP_SUPPORT_DIR, name.lstrip())
            else:
                line = ''
            if line and cleanup:
                line += "; %s" %cleanup

            # output timestamps after each line
            output_line = (
              "COMMAND_START=`{ts_cmd}`\n"
              "{line}\n"
              "COMMAND_END=`{ts_cmd}`\n"
              "echo 'COMMAND: {line}'\n"
              "echo COMMAND_RUNTIME: "
              "`echo \"$COMMAND_END-$COMMAND_START\" | bc` seconds\n"
              ).format(ts_cmd=TIMESTAMP_COMMAND, line=line)

            line = output_line
            # if the util type begins "local_" it goes to the local script.
            if exe.find("local_") == 0 or options.local_only:
                os.write(localfd, line)
            else:
                os.write(gatherfd, line)

        os.write(localfd, footer)
        os.close(localfd)

        if not options.local_only:
            os.write(gatherfd, footer)
            os.close(gatherfd)

    except OSError, (errno, strerror):
        error("OS error: %s" % os.strerror(errno))

def is_compliance():
    try:
        res = (sysctlbyname('isi.compliance') == 1)
    except Exception, e:
        res = False
    return res

#
# the core of this utility is basically isi_rdo, since that did
# the majority of what i wanted to do.
#
# assumptions:
#   - the script will be run on the quorum-side of the cluster (such
#   that ifs is writeable).
#   - the cluster has enough free space to store the temporary info
#   files and the final compressed package (about 500MB needed, usually
#   determined by the size of /var/log and the bug_infos).
#   - the cluster has enough wherewithal to actually gather the required
#   logs.
#

def main():
    global cluster, options, log, re_upload, IFS_READONLY

    activityIndicator = Dots()
    log = Logger(GATHER_STATUS_PATH)
    #
    # don't treat ^C special
    # and ignore the HUP signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)
    signal.signal(signal.SIGHUP, signal.SIG_IGN)

    # internal correctness check
    validateUtils()

    cluster.name     = None
    cluster.hostname = None
    cluster.user     = None
    cluster.password = None
    cluster.nodes    = []

    query           = False

    incremental_sequence = 0    # number of incremental gathers since full

    options.extra_file_list   = []
    options.gather_script     = []
    options.gather_expr       = []
    options.gather_expr_local = []
    options.analysis_script   = []
    options.analysis_expr     = []

    options.isi_nodes_args  = []
    options.includes        = []
    options.incremental     = False
    options.local_only      = False
    options.skip_node_check = False
    options.symlinkdir      = None
    options.tardir          = None
    options.tarfile         = None
    options.tempdir         = None
    options.use_ifs         = True
    options.no_exempt_logs  = False
    options.debug           = False
    options.verbose         = False
    options.upload_package  = True

    options.group           = []
    options.group_utils     = []
    options.ftp_enable      = True
    options.ftp_host        = 'ftp.isilon.com'
    options.ftp_user        = 'anonymous'
    options.ftp_pass        = ' '
    options.ftp_path        = "/incoming"
    options.ftp_port        = 21
    options.ftp_proxy_host  = ''
    options.ftp_proxy_port  = ''
    options.try_active_ftp  = True
    options.try_passive_ftp = True

    options.http_enable     = True
    options.http_host       = ''
    options.http_path       = ''
    options.http_proxy_host = ''
    options.http_proxy_port = ''

    options.esrs             = True
    options.clean_cores      = False
    options.clean_dumps      = False
    options.dumps            = False
    options.cores            = False
    options.save_only        = False
    options.verify_upload    = False
    options.re_upload        = False
    options.upgrade_archive  = False

    #
    # Allow all directory and data creation to be accessed by the group users
    #
    os.umask(0007)

    #
    # parse options
    #
    try:
        get_options()
    except ValueError as e:
        error(str(e))

    if options.save_only:
        saveConfigOptions()
        sys.exit(0)

    if options.verify_upload:
        verifyConnection()
        sys.exit(0)

    if options.re_upload:
        reUploadPackage(re_upload)
        sys.exit(0)

    if options.use_ifs and cluster.password:
        print "\nNote: A password is only required with -L or -T.  Ignoring."

    # update paths
    update_support_dirs(options)
    writeGatherRunFile()

    if len(options.analysis_expr):
        error("post-gather analysis expressions not currently supported")

    #
    # Currently we do not support an incremental gather file on local
    # filesystems, only on /ifs.
    #
    if options.incremental and not options.use_ifs:
        error("Incremental gather supported only on /ifs, not local filesystem")


    # do not support --group with --incremental, as incremental only reall
    # makes sense in relation to a full log set.
    if options.incremental and options.group:
        error("Cannot specify both --incremental and --group")

    #
    # query for the password if files are not saved to IFS.
    # Note we query BEFORE attempting any remote node activity,
    # such as makeDirs() call below.
    #
    if cluster.password == None and not options.use_ifs \
           and not options.local_only:
        if IFS_READONLY:
            cluster.password = getpass.getpass(prompt="Root password is required when /ifs is " \
                                   "a Read-only file system.\nPassword:")
        else:
            cluster.password = getpass.getpass()

    # first, verify that the node we're being run from is part of the
    # quorum-side of the cluster.
    #
    if options.use_ifs and not hasQuorum():
        error("This node does not have quorum -- OneFS is not writeable.  Aborting.")

    # Should we verify SUPPORT_DIR accessibility here?

    cluster.name, cluster.hostname = getHostname()

    #
    # figure out which nodes to run on.  first, though, determine which
    # nodes may be down.  we do this so that the customer need not
    # supply the -x semantic on the command line, since this may be
    # difficult for the customer to know or understand, especially in
    # larger clusters.
    #
    # since the system reports down devices based on device ID, but the
    # isi_rdo logic for -x uses LNN, we need to map b/w ID and
    # LNN.
    #
    if options.skip_node_check:
        print "Skipping checks for node availability."
    else:
        gi = group.GroupInfo()
        down_node_ids = gi.getDownNodeList()
        down_node_lnns = map(gi.getLnn, down_node_ids)
        if down_node_lnns:
            for nl in down_node_lnns:
                print "WARNING: node %s is down, skipping..." % nl
                options.isi_nodes_args += ['-x', str(nl)]

    cluster.nodes = runIsiNodes(options.isi_nodes_args)
    if not cluster.nodes:
        error("Invalid node list.")

    #
    # Save the invoked command line to the tarball
    #
    saveCommandLine()

    #
    # Get isi_firmware_status output. That command accepts the same
    # set of isi_nodes args, so running on the specified nodes is easy.
    #
    # NOTE: potentially unsafe, commented out until resolution of bug 43316.
    #       Fixed by bug 82479
    runIsiFirmwareStatus(options.isi_nodes_args)

    #
    # Convert -f filenames to UTILITIES commands; Add to UTILITIES
    # Convert gather_script/_expr to UTILITIES commands; Add to UTILITIES
    #
    extra_includes = []
    extra_includes.extend(addExtraFiles(options.extra_file_list))
    if not is_compliance():
        extra_includes.extend(addExtraScripts(options.gather_script))
        extra_includes.extend(addExtraExprs(options.gather_expr))
        extra_includes.extend(addExtraLocalExprs(options.gather_expr_local))

    #
    # Now create include_list using the final updated UTILITIES.
    # NOTE this step MUST follow the above AddExtraFoobar calls.
    # We also need to add any new extra_includes to the include_list
    # (although only if includes isn't "all").
    #
    if "all" in options.includes:
        include_list = UTILITIES.keys()
    else:
        include_list = options.includes
    include_list.extend(extra_includes)

    #
    # write the temp script that contains the instructions for information
    # gathering.  we're going to write this in TMP_SUPPORT_DIR,
    # so we need to 1) see if that dir already exists, and 2) create it
    # (and any parents) if it doesn't exist.
    #
    # note that the gathered information from the nodes is temporary,
    # and will be removed at the end of the script.  however, the final
    # package file (stored in PKG_SUPPORT_DIR, or specified tarfile path)
    # will be uniquely preserved.
    #
    # Note we need to make these directories on all nodes using ssh,
    # if we're not using /ifs.
    #
    # I'm purposefully not honoring the -n option when creating the node
    # directories -- all directories will be created -- on all nodes.
    #
    # NOTE: The paths created here for the nodes MUST match those
    #       utilized in the generate_scripts routine.
    dirs = [SUPPORT_DIR, TMP_SUPPORT_DIR, PKG_SUPPORT_DIR,
            LOCAL_TMP_SUPPORT_DIR]
    for n, a in cluster.nodes:
        nodedir = TMP_SUPPORT_DIR + n
        dirs.append(nodedir)
    count = makeDirs(dirs, options.use_ifs, options.local_only)
    if count:
        error("FAILED to make required directories on %d nodes." % count)


    try:
        last_full_gather = int(os.stat(FULLGATHER_FILE).st_mtime)
    except:
        last_full_gather = 0

    if options.incremental:
        if os.path.exists(INCREMENTAL_FILE):
            # failures here may reset the incremental sequence to 0
            try:
                incfile = open(INCREMENTAL_FILE, 'r')
                incdata = incfile.read()
                incfile.close()

                # this file may have placeholder data from previous
                # versions.
                if incdata.isdigit():
                    incremental_sequence = int(incdata)
            except OSError:
                pass

            incremental_sequence += 1
        else:
            sys.stderr.write("WARNING: No previous run recorded. ")
            sys.stderr.write("Falling back to non-incremental gather.\n")
            options.incremental = False

    #
    # if --cores/--dumps/--upgrade-archive specified explicitly than include
    # cores/dumps/upgrade_archive in the package
    #
    if options.cores:
        include_list.append('cores')
    if options.dumps:
        include_list.append('isi_hangdump')
    if options.upgrade_archive:
        include_list.append('upgrade_archive')

    #
    # build the gather script, based on the known information gathering
    # utilities, and the utilities the user is wishing to include
    #
    run_utils = buildRunUtils(include_list)
    if len(run_utils) == 0:
        error("No commands or info found to gather")

    generate_scripts(run_utils)
    #
    # run everything.
    #
    start_time=int(time.time())
    exit_status = 0

    print
    print "This may take several minutes.  Please do not interrupt the script."
    print

    activityIndicator.start()
    log.write("Gather In Progress on local node. Running.")
    task = "Running gather script on local node"

    (exit_code, output) = procs.get_cmd_output("bash " + LOCAL_GATHER_SCRIPT)

    # Save script output to a file
    with open("%s.output" % LOCAL_GATHER_SCRIPT, "w") as fp:
        fp.write("\n".join(output))

    if exit_code:
        print "ERROR %s" % task
        log.write("Gather Failed. %s" % output)
        for line in output:
            print line
    #
    # Save package xml metadata
    #
    pkginfo = {
        "gather_date": start_time,
        "last_full":   last_full_gather,
        "sequence":    incremental_sequence,
    }
    # if this generation fails, we should continue, but support
    # would like to see the traceback, and requested that we just
    # dump it into the file where the xml data should be.
    try:
        savePackageInfo(pkginfo, PACKAGE_INFO)
        os.link(PACKAGE_INFO, CELOG_EVENTS_INFO)
    except Exception, e:
        try:
            traceback.print_exc(file=open(PACKAGE_INFO, 'a'))
        except:
            pass

    post_args = []

    if is_compliance():
        post_args += ['sudo']

    post_args += ['isi_gather_info_slave']
    post_args += ['-j', '%d-%d' % (int(array.getLocalNode().lnn()), os.getpid())]
    if options.verbose:
        post_args += ['--verbose']

    if cluster.user is not None:
        post_args += ['-u', cluster.user]

    if cluster.password is not None:
        post_args += ['-p', cluster.password]

    if not options.local_only:
        #
        # Report any problems up-front.
        #
        task = "Running script on all cluster nodes"
        verbose("%s..." % task)
        exit_status, error_count, exit_results = runCommandOnNodes(post_args)
        if error_count:
            exit_status = 1
            print "ERROR %s (on %d nodes)." % (task, error_count)
            log.write("Gather Failed. %s (%s)" % (exit_results, error_count))

        #
        # If we're not using /ifs, we need to collect gather script
        # results from all nodes under test.
        #

        if not options.use_ifs:
            task = "Collecting script results from all cluster nodes"
            verbose("%s..." % task)
            if copyResultsFromNodes():
                exit_status = 1
                print "ERROR %s" % task
                log.write("Gather Failed. (%s)"  % task)

    #
    # run any analysis scripts
    # NOTE We're not dealing with options.analysis.expr just now
    #
    task = "Analyzing script results from all cluster nodes"
    if len(options.analysis_script):
        verbose("%s..." % task)
        if analyzeResultsFromNodes():
            exit_status = 1
            print "ERROR %s" % task
            log.write("Gather Failed. (%s)" % task)

    #
    # put everything into one tarball
    #
    log.write("Gather Succeeded.")
    print "Information gathering completed... creating compressed package."
    pkg_path_name = makePkgName()

    try:
        tar_command = "/usr/bin/tar czf %s . 2>&1" % pkg_path_name
        os.chdir(TMP_SUPPORT_DIR)
        (exit_code, output) = procs.get_cmd_output(tar_command)
        if exit_code:
            print "ERROR tar returned %d" % (exit_code >> 8),
            log.write("Gather Failed")
            if output:
                print ", output follows:"
                for line in output:
                    print line
            else:
                print ""
            error("Could not create compressed package.")
            log.write("Gather Failed. Could not create compressed package.")
    except OSError, (errno, strerror):
        error("FAILED system call (tar): %s" % (os.strerror(errno)))
        log.write("Gather Failed %s" % (os.strerror(errno)))

    if options.symlinkdir:
        (head, tail) = os.path.split(pkg_path_name)
        symlink_path = os.path.join(options.symlinkdir, tail)
        try:
            os.symlink(pkg_path_name, symlink_path)
        except OSError, e:
            errstr = "Failed to symlink %s to %s: %s" % (pkg_path_name,
                    symlink_path, e)
            error(errstr)
            log.write(errstr)


    activityIndicator.join()
    print "Packaging complete..."
    print "Package: %s" % pkg_path_name
    log.write("Package: %s" % pkg_path_name)

    #
    # If we don't have the exit_status set, then the log gathered
    # completely, and we should create the touchfile for incremental log
    # gathers
    #
    if not exit_status:
        try:
            incfile = open(INCREMENTAL_FILE, 'w')
            incfile.write(str(incremental_sequence))
            incfile.close()
        except:
            sys.stderr.write('WARNING: Incremental run data not recorded.\n')
        try:
            os.utime(INCREMENTAL_FILE, (start_time, start_time))
        except:
            sys.stderr.write('WARNING: Incremental run data update failed.\n')
            log.write("Gather Failed. WARNING: Incremental run data update failed.")

        if not options.incremental:
            if not os.path.exists(FULLGATHER_FILE):
                try:
                    fullgather = open(FULLGATHER_FILE, 'w')
                    fullgather.write('This space intentionally left blank.\n')
                    fullgather.close()
                except:
                    sys.stderr.write('WARNING: Full Gather run data not recorded.\n')

            try:
                os.utime(FULLGATHER_FILE, (start_time, start_time))
            except Exception, e:
                sys.stderr.write('WARNING: Full Gather run data update failed. %s\n' % str(e))
                log.write("Gather Failed. WARNING: Full Gather run data update failed.")

    if options.upload_package:
        print "Uploading in progress.  If problems are encountered during the"
        print "upload process, the package will need to be sent manually."
        uploadPackage(pkg_path_name)
    else:
        print "No Upload method selected. Package not uploaded."
        log.write("No Upload method selected. Package not uploaded.")

    print "Cleaning up temporary data...",
    cleanupGatherRunFile()
    if clean_tmp_dir(TMP_SUPPORT_DIR) == 0:
        print "done."

    verbose("\nCleaning up core/dumps")
    if options.clean_cores:
        if clean_cores(True) == 0:
            print "core clean done."
    if options.clean_dumps:
        if clean_cores(False) == 0:
            print "dump clean done."

    log.join()
    sys.exit(exit_status)

def runIsiNodes(args):
    isi_nodes_specifiers = '%{id} %{internal} %{node} %{lnn}'
    nodes = []

    # Add a column label line
    (exit_code, output) = procs.get_cmd_output('echo %s > %s' % \
                                          (isi_nodes_specifiers, \
                                           ISI_NODES_OUTPUT))
    if exit_code:
        if len(output):
            print output[0]
        else:
            print "Error writing file", ISI_NODES_OUTPUT
        return None

    (exit_code, output) = procs.get_cmd_output('isi_nodes %s %s >> %s' % \
                                          (string.join(options.isi_nodes_args),\
                                           isi_nodes_specifiers, \
                                           ISI_NODES_OUTPUT))
    if exit_code:
        if len(output):
            print output[0]
        else:
            print "Error running isi_nodes."
        return None

    isi_nodes_file = open(ISI_NODES_OUTPUT, 'r')

    # Skip column label line
    isi_nodes_file.readline()
    for line in isi_nodes_file:
        toks = line.split()
        # tuple of %{node} and %{ip_address} from isi_nodes
        nodes.append((toks[2], toks[1]))

    isi_nodes_file.close()

    return nodes

def saveCommandLine():
    try:
        mode = os.O_RDWR|os.O_CREAT|os.O_TRUNC
        fd = os.open(SAVED_COMMAND_LINE, mode, 0666)
        os.write(fd, " ".join(sys.argv) + "\n")
        os.close(fd)
    except OSError, e:
        error("Failed saving command-line output")

def savePackageInfo(pkginfo, fname):
    # this saves to the package information about the package

    contents = []
    if options.no_exempt_logs:
        contents.append('none')
    elif options.incremental:
        contents.append('incremental')
    else:
        contents.append('full')

    for group in options.group:
        contents.append("group:%s" % group)

    # Bug 167340 - switch to git attr ident
    # Use \x24 instead of $ to avoid git interpolation
    revision = re.match(r"\x24Id: ([\da-f]+) \x24", REVISION).group(1)

    # generate our xml
    pkg = etree.Element("package")
    pkg.attrib["version"] = "1"

    etree.SubElement(pkg, "command").text     = " ".join(sys.argv)
    etree.SubElement(pkg, "source_lnn").text  = str(getLocalLNN())
    etree.SubElement(pkg, "contents").text    = ",".join(contents)
    etree.SubElement(pkg, "gather_date").text = str(pkginfo["gather_date"])
    etree.SubElement(pkg, "revision").text    = revision

    logs = etree.Element("logs")
    etree.SubElement(logs, "last_full").text  = str(pkginfo["last_full"])
    etree.SubElement(logs, "sequence").text   = str(pkginfo["sequence"])
    pkg.append(logs)

    # Bug 158435 - append the saveCelogEvents() XML

    # this saves to the package identifying information about the
    # cluster.  all the information we want is directly generated
    # by celog events, so we're just reusing it.

    # XXX this section is going to change to be more
    # self contained in the celog libraries.
    cfg = isi.celog.celog_config_api.CelogConfigAPI()
    custcfg = cfg.get_contact_settings()

    guid = arrcfg.config_load().guid

    ev = isi.celog.attachments.AllEvents([])
    ev.custcfg = custcfg
    ev.guid = guid
    evxml = ev.get_evxml(ev.events)
    pkg.append(evxml)

    f = open(fname, 'w')
    f.write(etree.tostring(pkg))
    f.close()


def runIsiFirmwareStatus(args):
    cmd = ('/usr/bin/isi_hwtools/isi_firmware_status --cluster --dump %s > %s' % \
           (' '.join(args), ISI_FIRMWARE_OUTPUT))
    if is_compliance():
        cmd = 'sudo ' + cmd
    (exit, output) = procs.get_cmd_output(cmd)
    if exit:
        print 'Error running "isi_firmware_status"'
        for line in output:
            print line

def waitForPid(pid):
    while True:
        pid, status = os.waitpid(pid, 0)
        if os.WIFEXITED(status):
            return os.WEXITSTATUS(status)
        elif os.WIFSIGNALED(status):
            return 1

def waitForChildren(children):
    #
    # NOTE using max(exit_results) could miss potential negative returns,
    # masked by (greater than) 0 success results.  This code will still
    # get max, ignoring all 0s, returning negatives if caught and "max".
    #
    exit_results = [waitForPid(child) for child in children]
    exit_errors = filter(lambda x: x!=0, exit_results)
    error_count = len(exit_errors)
    exit_status = error_count and reduce(max, exit_errors) or 0
    return(exit_status, error_count, exit_results)

def hasQuorum():
    try:
        fp = os.popen("sysctl -n efs.gmp.has_quorum", 'r')
    except OSError, (errno, strerror):
        error("popen failed on efs.gmp.has_quorum: %s" % os.strerror(errno))

    quorum = fp.read()
    fp.close()
    quorum = quorum.strip('\n')

    if quorum == "1":
        return True
    elif quorum == "0":
        return False
    else:
        print "Unknown return from efs.gmp.has_quorum: %s" % quorum
        return False

def getLocalLNN():
    try:
        fp = os.popen("/usr/bin/isi_nodes -L %{lnn}", 'r')
        isi_nodes = fp.read()
        fp.close()
        lnn = int(isi_nodes.strip('\n'))
    except:
        lnn = -1

    return lnn

def makeDir(path):
    global options
    if not os.path.exists(path):
        try:
            os.makedirs(path, 0770)
        except OSError, (errno, strerror):
            error("couldn't make %s directory: %s" % \
                (path, os.strerror(errno)))
    else:
        # Bug 189123 - If a temp directory has been specified, we don't want to
        # clobber it's permissions if it already exists.
        if not options.tempdir:
            os.chmod(path, 0770)

def getHostname():
    try:
        f = os.popen("/bin/hostname", "r")
        host_name = f.readline()
    except OSError, (errno, strerror):
        error("error running hostname: %s" % os.strerror(errno))
    f.close()

    host_name = host_name.strip()
    names     = host_name.split('-')

    # this code does :
    # host_name      len(names) final names:
    # something      1          something
    # something-2    2          something
    # some-thing-2   3          some-thing
    # some-th-ing-2  4          some-th-ing
    if len(names) > 1:
        names.pop()
    cluster_name = '-'.join(names)

    return cluster_name, host_name

def makePkgName():
    if options.tarfile != None:
        pkg_path_name = fixTarfileName(options.tarfile)
        if pkg_path_name != None:
            return(pkg_path_name)

    time_string = time.strftime("%Y%m%d-%H%M%S")
    pkg_name = "IsilonLogs-%s-%s.tgz" % (cluster.name, time_string)
    pkg_path_name = os.path.join(PKG_SUPPORT_DIR, pkg_name)

    return(pkg_path_name)

def makeDirs(dirs, use_ifs=False, local_only=False):
    '''Returns count of nodes that failed mkdir command.'''
    if use_ifs or cluster == None or cluster.nodes == None or local_only:
        for _dir in dirs:
            makeDir(_dir)
        return(0)
    cmd_args = []
    if is_compliance():
        cmd_args.append('sudo')
    cmd_args += ['isi_gather_info_slave', '-s']
    exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
    if error_count > 0:
        return (error_count)

    cmd_args = ['mkdir','-p'] + dirs
    exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)

    if error_count > 0:
        for _dir in dirs:
            print _dir

        print "\nPlease check above directories accessibilities in the below nodes..."
        for index in range(len(exit_results)):
            if exit_results[index]:
                print cluster.nodes[index][0]

    return(error_count)

def listGroups():
    print "\nKnown groups (included with the --group option): \n"
    groups = GROUPS.keys()
    groups.sort()
    for g in groups:
        print "  %s" % g
    print

def listUtils():
    print
    print "Known utilities (included with the -i option): \n"
    utils = UTILITIES.keys()
    utils.sort()
    for u in utils:
        if u not in EXEMPT:
            print "  %s" % u
    print

#
# this routine is simply here to print the warning message, and alert
# the user they munged the name.  i don't go through the pain of
# removing the unknown name from the include array, since it won't
# hurt anything being in there (since the keys of the UTILITIES dict
# is what's searched).
#
def pruneUtils(requested):
    ulist = UTILITIES.keys()
    nlist = []
    for r in requested:
        if not r in ulist:
            print "WARNING: unknown utility (%s), ignored..." % r
        else:
            nlist.append(r)

    return(nlist)

#
# An internal correctness check to verify that everything in GROUPS and
# EXEMPT actually exists.
#
def validateUtils():
    util_errors = 0
    for (group, utils) in GROUPS.items():
        for util in utils:
            if not UTILITIES.has_key(util):
                util_errors += 1
                print "Unknown utility '%s' in group '%s'!" % (util, group)
    for util in EXEMPT:
        if not UTILITIES.has_key(util):
            print "Unknown utility '%s' in default group!" % util
    if util_errors:
        print "%d utility errors." % util_errors
        sys.exit(1)

#
# the logic: anything (valid) in the include list automatically means
# all other utilities are excluded (except for things in EXEMPT).
# (unless user specified '--nologs', which skips EXEMPT)
#
def buildRunUtils(ilist):
    if ilist != None:
        _list = pruneUtils(ilist)
    else:
        _list = []
    if options.group_utils:
        _list.extend(options.group_utils)
    if options.no_exempt_logs == False:
        _list.extend(EXEMPT)
    return(_list)

#
# Given a user-supplied tarfile, fix it to ensure:
# o If full path specified, use that path
# o If no full path given, use original PKG_SUPPORT_DIR path
# o Check extension: if usual tarfile extension, make sure we
#   don't "double up" with additional .tgz; otherwise keep intact.
# o Also check for special case '/' (root) specified
# Returns None for any problems (such as '/' case)
#
def fixTarfileName(tarfile):
    tarfile = os.path.normpath(tarfile.strip())
    #
    # Note this verifies we were not given only '/'
    #
    if len(tarfile) and not (tarfile[0]=='/' and len(tarfile)==1):
        #
        # Is it a full path or not? If not, supply default path.
        #
        if tarfile[0] != '/':
            tarfile = PKG_SUPPORT_DIR + tarfile
        #
        # Does it have an extension already?
        #
        root,ext = os.path.splitext(tarfile)
        if ext in ['.tar','.tgz']:
            tarfile = root
        elif ext in ['.gz']:
            root2,ext2 = os.path.splitext(root)
            if ext2 in ['.tar']:
                tarfile = root2
            else:
                tarfile = root
        else:
            #
            # Note we'll just keep original extensions...
            #
            pass
        tarfile = tarfile + ".tgz"
        return(tarfile)
    else:
        return(None)

def getPathFile(pathfile):
    # NOTE we need this rstrip mod for reliable dirname/basename!
    x = pathfile.rstrip('/')
    return(os.path.dirname(x), os.path.basename(x))

def getUtilityGatherCmd(name, path, base):
    #
    # Avoid gathering duplicate data from all the nodes.
    # If path starts with /ifs, than gather data from only one node (local node)
    # e.g. path=/ifs/data or path=/ifs
    #
    if not options.local_only and len(path) \
            and ((path[0:4] == '/ifs') or (path == '/' and (name == 'ifs'))):
        c = ('local_tar', "cd %s" % (path), "%s.tar %s" % (name, base))
    else:
        c = ('tar', "cd %s" % (path), "%s.tar %s" % (name, base))
    return(c)

#
# 'cmd' better be executable - either fullpath or in PATH
#
def getUtilityExecuteCmd(name, cmd, args):
    c = ('exe', "%s %s" % (cmd, args), "%s" % (name))
    return(c)

def getUtilityExecuteLocalCmd(name, cmd, args):
    c = ('local_exe', "%s %s" % (cmd, args), "%s" % (name))
    return(c)

def makeUniqueDictKeyName(name, dictionary):
    new_name, count = name, 0
    while new_name in dictionary.keys():
        count += 1
        new_name = '%s-%d' % (name, count)
    return new_name

#
# Use a dict and we get unique non-repeated entries!
# (Since the key name is uniquely determined from given extra_list)
# Be sure to rstrip '/' from any dirs/cmds to get proper 'basename'.
# Be sure to lstrip '/' from any dirs/cmds/exprs to start names without
# extra baggage.  ===> rstrip + lstrip = just strip
# Get unique 'name' from stripping away nonalnum, or replacing.
#
def getExtraDict(extra_list, extra_entry_func):
    #
    # This is really just for the expressions:
    # Some additional common expression chars to replace
    # (by including '-' in this list, we'll reduce multi -'s)
    #
    replace_list = '\\|/.,;: <>-!@#$%&*()=+' + "'" + '"'
    retain_list = '_'

    extra = {}
    for e in extra_list:
        # use consistent strip of '/', see comments above
        old, new = list(e.strip('/')), []
        last_replace = False
        for o in old:
            if o.isspace() or o in replace_list:
                if not last_replace:
                    new.append('-')
                    last_replace = True
            elif o.isalnum() or o in retain_list:
                new.append(o)
                last_replace = False
        name = ''.join(new)
        name = makeUniqueDictKeyName(name,extra)
        extra[name] = extra_entry_func(e)
    return(extra)

def addExtraFiles(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        extra_dict = getExtraDict(extra_list, lambda e: getPathFile(e))
        for name,(path,base) in extra_dict.iteritems():
            # Don't gather /ifs/data/Isilon_Support
            if (os.path.join(path, base) in ('/ifs/data/Isilon_Support',
                '/ifs/data/Isilon_Support/pkg', '/ifs/data/Isilon_Support/tmp')):
                print "WARNING: ignored path %s/%s" % (path, base)
                continue
            UTILITIES[name] = getUtilityGatherCmd(name,path,base)
            extra_includes.append(name)
    return(extra_includes)

#
# NOTE that simply adding these scripts to the include list isn't sufficient;
# we need to ensure they exist on the nodes, or are copied there if necessary.
# However, it is SIMPLER to leave this requirement up to the user or calling
# script. Perhaps isi_for_array or isi_scp.
#
def addExtraScripts(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        #
        # Future enhancement possibility:
        # Do we want user-supplied options, per-script?
        # will quotes work with getopt.getopt?
        # i.e. -s "scriptname --opt1 opt2" -otheroption ...
        # ==> (cmd,args) = (scriptname, "--opt1 opt2")
        #
        extra_dict = getExtraDict(extra_list, lambda e: (e, ''))
        for name,(cmd,args) in extra_dict.iteritems():
            UTILITIES[name] = getUtilityExecuteCmd(name,cmd,args)
            extra_includes.append(name)
    return(extra_includes)

def addExtraExprs(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        # leave e intact! (in the returned extra dict cmd)
        extra_dict = getExtraDict(extra_list, lambda e: (e, ''))
        for name,(cmd,args) in extra_dict.iteritems():
            UTILITIES[name] = getUtilityExecuteCmd(name,cmd,args)
            extra_includes.append(name)
    return(extra_includes)

def addExtraLocalExprs(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        extra_dict = getExtraDict(extra_list, lambda e: (e, ''))
        for name,(cmd,args) in extra_dict.iteritems():
            UTILITIES[name] = getUtilityExecuteLocalCmd(name,cmd,args)
            extra_includes.append(name)
    return(extra_includes)

def copyScriptToNodes(script):
    result = 0
    for node_name, node_address in cluster.nodes:
        dest = "%s:%s" % (node_name, script)
        scp_error = isi_scp_main(script, dest, cluster.user, cluster.password)
        if scp_error:
            error('Error copying script to %s' % node_name)
            # Note error is sys.exit... we currently won't actually return
            result += 1
    return(result)

#
# Use -r to recursively grab entire remote directory structure
# Keep in mind: GATHER_SCRIPT = TMP_SUPPORT_DIR + '_gather'
# So all the results are in TMP_SUPPORT_DIR, and in fact the
# commands executed spit results into TMP_SUPPORT_DIR$HOST/something
# So essentially we want to grab (scp -r) TMP_SUPPORT_DIR from
# each node, which would (if properly done at exec time) only have
# a node-unique $HOST dir in existence that would be copied to
# the main node, not interfering with any other node results.
#
# Note that using scp -r, we have to be careful:
# (1) we can't scp from remote:/path/tmp/ to local:/path/tmp/
#     because you end up with results in: local:/path/tmp/tmp/...
# (2) above problem is dealt with by using parent dir, i.e.,
#     scp from remote:/path/tmp to local:/path/
# (3) HOWEVER, we need to make sure we never do this when
#     remote host == local host (to self), to avoid recursion problems
#
def copyResultsFromNodes():
    result = 0
    dest, dirname = getPathFile(TMP_SUPPORT_DIR)
    for node_name, node_address in cluster.nodes:
        #
        # Don't copy from self to self:
        #
        if node_name == cluster.hostname:
            continue
        src = "%s:%s" % (node_name, TMP_SUPPORT_DIR)
        scp_error = isi_scp_main(src, dest, cluster.user, cluster.password,
                                 options=['-r'])
        if scp_error:
            error('Error copying results file from %s' % node_name)
            # Note error is sys.exit... we currently won't actually return
            result += 1
    return(result)

#
# Need to pass two arguments to script:
#    arg 1: full directory path location of results
#    arg 2: cluster_name
# With these two args, any script should be able to parse the results dir,
# find appropriate node subdirs (based on hostname). It is up to the script
# to determine what is there... It should not assume each dir is a results
# dir, nor should it make any assumptions about files in the top dir --
# These may be results of other analysis scripts that have previously run.
# The scripts can put any output or output logs into this directory --
# Though beware of overwriting any existing files!
# After all analysis scripts have run, everything in this top dir down will
# be tarred up into the final tarball.
#
def analyzeResultsFromNodes():
    results = 0
    results_dir = TMP_SUPPORT_DIR
    script_output = {}
    for script in options.analysis_script:
        verbose("\nRunning: %s" % script)
        name = script.strip('/').replace('/','-').replace('.','-')
        name = makeUniqueDictKeyName(name,script_output)
        script_output[name] = "%s%s.out" % (results_dir, name)
        command = "%s %s %s" % (script, results_dir, cluster.name)
        (exit_code, output) = procs.get_cmd_output(command)
        if output and len(output):
            try:
                fd = os.open(script_output[name],
                             os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0666)
                for line in output:
                    os.write(fd, "%s\n" % line)
                    #
                    # Changed from original:  verbose(line)
                    # We want this output to be seen to report any analysis
                    # output, always.
                    #
                    print line
                os.close(fd)
            except OSError, (errno, strerror):
                results += 1
                print "FAILED %s: %s" % (script, os.strerror(errno))
        if exit:
            results += 1
            print "FAILED %s: returned %d" % (script, exit >> 8)
    return(results)

def execCommandOnNodes(post_args):
    children = []
    pre_args = ['isi_rdo']
    for node_name, node_address in cluster.nodes:
        child = os.fork()
        if not child:
            if cluster.user != None:
                pre_args += ['-u', cluster.user]
            if cluster.password != None:
                temp_file = tempfile.TemporaryFile('w+b', -1, 'isi_for_array')
                temp_file.write(cluster.password)
                temp_file.flush()
                temp_file.seek(0)
                #
                # as of Python 2.3, TemporaryFile fd's are not inherited
                # by child procs so dup it for the exec below.
                #
                dupfd = os.dup(temp_file.fileno())
                temp_file.close()
                pre_args += ['-q', '/dev/fd/%d' % dupfd]
            pre_args += [node_address]
            os.execvp('isi_rdo', pre_args + post_args)
        else:
            children.append(child)
    return children

def runCommandOnNodes(cmd_args):
    '''Returns tuple (exit_status, error_count, exit_results):
       exit_status : max non-zero exit_result value
       error_count : number of non-zero exit status in exit_results
       exit_results: array of per-child exit status
    '''
    children = execCommandOnNodes(cmd_args)
    return waitForChildren(children)

def isi_scp_main(file1, file2, user=None, password=None, options=None):
    '''isi_scp [OPTION]... [[USER@]HOST1:]FILE1 [[USER@]HOST2:]FILE2'''
    cmd_args = ['isi_scp']
    children = []
    child = os.fork()
    if not child:
        if user != None:
            cmd_args += ['--user', user]
        if password != None:
            # change to isi_scp pre later? for association?
            temp_file = tempfile.TemporaryFile('w+b', -1, 'isi_for_array')
            temp_file.write(cluster.password)
            temp_file.flush()
            temp_file.seek(0)
            #
            # as of Python 2.3, TemporaryFile fd's are not inherited
            # by child procs so dup it for the exec below.
            #
            dupfd = os.dup(temp_file.fileno())
            temp_file.close()
            cmd_args += ['-q', '/dev/fd/%d' % dupfd]
        if options:
            cmd_args += reduce(lambda a,b: a+['-o']+['%s' % b], options, [])
        cmd_args += [ file1, file2 ]
        os.execvp('isi_scp', cmd_args)
    else:
        children.append(child)
    exit_status, error_count, exit_results = waitForChildren(children)
    return exit_status


def attempt_esrsv3_upload(package_path):
    ''' string -> bool

    try to upload the package with ESRSv3 MFT PUT, attempts to report an
    intelligent error message if ESRS is misconfigured, disconnected or the
    upload itself fails
    '''
    swid = licensing.get_swid()
    msg = 'an unexpected error occurred'

    try:
        msg = 'The cluster could not communicate with the ESRSv3 gateway'
        connectivity = esrs.pcm_get_connectivity_status(swid)

        if connectivity == 'null':
            msg = 'ESRSv3 has not been provisioned'
            raise esrs.ESRSError

        msg = "Upload to ESRS via REST failed"
        esrs.mft_upload_file(swid, package_path, useragent='isi_gather_info')

    except (esrs.ESRSCurlError, esrs.ESRSError, esrs.ESRSBadCodeError) as err:
        # something went wrong, 'msg' will have the value of the last thing
        # that we were about to do
        msg += ' error: ' + str(err)
        print(msg)
        traceback.print_exc()
        log.write(msg)
        return False

    return True


# this routine is responsible for uploading the package to isilon.
# it will try any supported access methods (currently esrs, http, ftp), or
# as specified on the command line.
def uploadPackage(package_path):
    global options
    global log
    uploaded = False

    # first esrs
    if options.esrs:

        # open ESRSv3 configuration
        v2tree = gconfig.Tree("esrs")
        v2base = gconfig.open(v2tree)
        v2ctx = v2base.ctx_new()
        esrs_cfg = v2ctx.read("esrs")

        # open ESRS v2 (legacy) configuration
        v1tree = gconfig.Tree("remote-support")
        v1base = gconfig.open(v1tree)
        v1ctx = v1base.ctx_new()
        connectemc_cfg = v1ctx.read("connectemc")

        # try ESRSv3
        if esrs_cfg["enabled"]:
            # fetch licensing SWID, which is used as the cluster ID
            uploaded = attempt_esrsv3_upload(package_path)

            if uploaded:
                msg = 'Package successfully uploaded with ESRSv3'
            else:
                msg = 'ESRSv3 upload failed. See previous messages for cause.'

            print msg
            log.write(msg)

        # check if ESRSv2 is enabled
        elif connectemc_cfg["enabled"]:
            esrs_host = connectemc_cfg["esrs_host"]

            for gateway in ("primary_esrs_gateway", "secondary_esrs_gateway"):
                if not connectemc_cfg[gateway] or uploaded:
                    continue

                try:
                    curl_error = send_file_to_esrs_via_curl(
                        esrs_host, connectemc_cfg[gateway], "8118",
                        "/r/mft/incoming", package_path)

                    # curl returns error code 0 if connected
                    if curl_error == 0:
                        # uploaded successfully
                        uploaded = True
                        msg = 'Package successfully uploaded with ESRSv2'
                        print msg
                        log.write(msg)

                except Exception as e:
                    msg = "Upload Failed to HTTP host: [%s]. Error: %s" % \
                          (esrs_host, str(e))
                    print msg
                    traceback.print_exc()
                    log.write(msg)

    #then ftp
    if options.ftp_enable and not uploaded:
        if not uploaded and options.try_passive_ftp:
            print "Trying Passive FTP..."
            log.write("Upload in Progress. (FTP - Passive Upload).")
            try:
                uploaded = uploadPackageFTP(package_path, pasv=True)
            except ftplib.all_errors, e:
                msg = "Upload Failed to FTP host: [%s]. (Passive Upload)."\
                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)
            except Exception, e:
                msg = "Upload Failed to FTP host: [%s]. (Passive Upload)." \
                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)

            if uploaded:
                msg = "Uploaded Succeeded (FTP - Passive). File %s" % \
                    os.path.basename(package_path)
                print msg
                log.write(msg)

        if not uploaded and options.try_active_ftp:
            print "Trying Active FTP..."
            log.write("Upload in Progress. (FTP - Active Upload).")
            try:
                uploaded = uploadPackageFTP(package_path, pasv=False)
            except ftplib.all_errors, e:
                msg = "Upload Failed to FTP host: [%s]. (Active Upload)."\
                                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)
            except Exception, e:
                msg = "Upload Failed to FTP host: [%s]. (Active Upload)."\
                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)

            if uploaded:
                msg = "Upload Succeeded (FTP - Active). File %s" % \
                                os.path.basename(package_path)
                print msg
                log.write(msg)

    #then http
    if options.http_enable and options.http_host and not uploaded:
        try:
            client = MyHttp(options.http_host, options.http_path, \
                            options.http_proxy_host, options.http_proxy_port)
            msg = "Upload in Progress. (HTTP). Filename: %s" % \
                os.path.basename(package_path)
            print msg
            log.write(msg)

            package = file(package_path,'r')
            (exit_code, ftpError) = client.storeFile(package)
            if exit_code:
                uploaded = False
                raise Exception(ftpError)
            else:
                uploaded = True
                msg ="Upload Succeeded. (HTTP). Filename: %s" % \
                          os.path.basename(package_path)
                print msg
                log.write(msg)
        except Exception, e:
            msg = "Upload Failed to HTTP host: [%s]. Error: %s" % \
                (options.http_host, str(e))
            print msg
            log.write(msg)

    if not uploaded:
        msg = "Package upload failed with all protocols please upload it manually."
        log.write(msg)
        print(msg)
    return uploaded


def forArrayCmd(nodeOps, args):
    proxy = rpc.RPCDaemonClusterProxy(nodeOps,
            'isi.app.lib.procs.RunCmdOutput')
    result = proxy.run(args, raw = True)
    sorted_keys = result.keys()
    sorted_keys.sort(lambda x, y: cmp(x.lnn(), y.lnn()))
    msglist = []
    for key in sorted_keys:
        if result[key][1]:
            msglist.append('%s:\n' % key.name())
            msglist.append(str(result[key][1]))
    return ''.join(msglist)


def uploadPackageFTP(package_path, pasv=True):
    global options
    global log

    package_filename = os.path.basename(package_path)
    package = file(package_path, "r")

    client = MyFTP(options.ftp_host, options.ftp_user, options.ftp_pass, \
                   options.ftp_path, options.ftp_port, options.ftp_proxy_host, \
                   options.ftp_proxy_port)
    client.set_pasv(pasv)
    try:
        response = client.storbinary("STOR %s" % package_filename, package)
        if '226' in response:
            log.write("Upload Succeeded. File %s" % package_filename)
            return True
        else:
            msg = "Upload Failed to FTP host: [%s]. Response: %s" % (options.ftp_host, response)
            print msg
            log.write(msg)
            return False
        print       #flushes the sys.stdout buffer and start newline
        try:
            client.quit()
            client.close()
            package.close()
        except ftplib.all_errors:
            # ftp.isilon.com drops connections in a violent way
            # when client.quit() is called, which causes ftplib to
            # raise an exception.
            # Ignore any errors at this point, because we've
            # completed the upload if we've reached this far,
            # and we simply don't care about the rest.
            pass
            return True

    except Exception, e:
        msg = "Upload Failed to FTP host: [%s]. Response: %s" % (options.ftp_host, str(e))
        print msg
        log.write(msg)
        return False

def reUploadPackage(item):
    archivePath = PKG_SUPPORT_DIR#IFS_SUPPORT_DIR+_PKG_SUPPORT_DIR
    itempath = os.path.realpath(item)
    flock = MyLock()

    try:
        #is it a complete path?
        if os.path.exists(itempath):
            fname = itempath
            uploadPackage(fname)
        #is it a file? then attach archives path
        elif os.path.exists(os.path.join(archivePath,item)):
            fname = os.path.join(archivePath, item)
            uploadPackage(fname)
        else:
            flock.unlock()
            error("Log file: %s could not be found" % item)
    except Exception, e:
        error("Error Re-Uploading file %s" % str(e))
        flock.unlock()
    flock.unlock()

def verifyConnection():
    def sigTimerHandler(self, signum, frame):
        raise Exception ("Connection Verification Failed: Connection Timed Out.")

    global log
    import tarfile
    from cStringIO import StringIO

    log.write("Connection Verification in Progress")
    print "Connection Verification in Progress"

    ##create tar file###
    path = '/tmp/'
    fname = 'isiVerifyFile-' + time.strftime('%Y%m%d-%H%M%S') + '.tar.gz' #salt
    myTarFile = tarfile.open(path + fname, 'w:gz')
    try:
        try:
            data = "DEADBEEF"*100
            info = tarfile.TarInfo('garbagefile')
            info.size = len(data)
            myTarFile.addfile(info, StringIO(data))
        except Exception, e:
            print "Connection Verification (HTTP) Failed: Tarfile" \
            "creation error. %s" % str(e)
            log.write("Connection Verification (HTTP) Failed: Tarfile" \
            "creation error. %s" % str(e))
            sys.exit(0)
    finally:
        myTarFile.close()
   #/create tar file###

    #CAVEAT: can wreak havoc if multiple procs are in process
    signal.signal(signal.SIGALRM, sigTimerHandler)
    signal.alarm(300)       #5-minute timeout
    try:
        success = uploadPackage(path+fname)
        if success:
            print "Connection Verification Succeeded."
            log.write("Connection Verification Succeeded.")
        else:
            print "Connection Verification Failed."
            log.write("Connection Verification Failed.")
    except Exception, e:
        log.write("Connection Verification Failed. %s" % str(e))
        print "Connection Verification Failed. %s" % str(e)
        pass

    try:
        os.remove(path+fname)
    except:
        print "Error in removing temporary file: %s " % str(path+fname)
        log.write("Error in removing temporary file: %s " % str(path+fname))
        pass
    else:
        print "Temp verify file removed."

    sys.exit(0)
    return

def verbose(message):
    if options.verbose:
        print >> sys.stdout, message

def error(message):
    print >> sys.stderr, PROGNAME + ':', message
    sys.exit(1)

def clean_tmp_dir(directory):
    '''Function cleans up all temporary data on all nodes.  Returns the number
        of failures returned from executing rm.'''
    error_count = 0
    cmd_args = ["/bin/rm", "-rf", directory]
    if options.use_ifs or options.local_only:
        cmd = " ".join(cmd_args)
        (exit_code, output) = procs.get_cmd_output(cmd)
        if exit_code:
            error_count += 1
            print "\nERROR cleaning up %s, output follows:\n%s" % (directory, \
                                                        "\n".join(output))
    else:
        exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
        if error_count:
            print "\nERROR cleaning up %s on %d nodes." % (directory, error_count)
    if error_count:
        print "You may wish to delete these files manually."
    return error_count

def clean_cores(isCore=True):
    '''Function cleans up all cores/dumps on all nodes.  Returns the number
        of failures returned from executing rm.'''
    error_count = 0
    if isCore:
        cmd_args = ["/bin/rm", "-rf", "/var/crash/*.core.gz"]
        ndmp_cmd_args = ["/bin/rm", "-rf", "/ifs/.ifsvar/modules/ndmp/core/*.core.gz"]
    else:
        cmd_args = ["/bin/rm", "-rf", "/var/crash/isi_hangdump*"]
    if options.local_only:
        cmd = " ".join(cmd_args)
        (exit, output) = procs.get_cmd_output(cmd)
        if exit:
            error_count += 1
            print "\nERROR cleaning up %s, output follows:\n%s" % (dir, \
                                                        "\n".join(output))
    else:
        exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
        if error_count:
            print "\nERROR cleaning up %s on %d nodes." % (dir, error_count)
    if error_count:
        print "You may wish to delete these files manually."

    # clean up ndmp cores
    if isCore:
        cmd = " ".join(ndmp_cmd_args)
        (exit, output) = procs.get_cmd_output(cmd)
        if exit:
            print "You may wish to delete ndmp cores manually."

    return error_count

class MyHttp(httplib.HTTPConnection):
    """extends httplib.HTTPConnection with a storeFile that has a callback
        function. the callback function intercepts the status of the file
        transfer. a ProgressBar implementation that prints to stdout.
    """
    def __init__(self, host='', path='/put/', proxy='', proxyport=''):
        self.startTime = 0
        self.fileSize = 0
        self.uploaded = 0
        self.updateLogIntVal = 3                #interval to log to file
        self.updateTtyIntVal = 1                #interval to update progress
        self.lastLogTime = 0
        self.lastTtyTime = 0
        self.pbar = ''                          #ProgressBar
        self.enableLog = True
        global log
        self.log = log

        if proxy and proxyport:
            httplib.HTTPConnection.__init__(self, host=proxy, port=proxyport, \
                                            strict=0)
        else:
            httplib.HTTPConnection.__init__(self, host=host, port=80, strict=0)
        addrInfo = socket.getaddrinfo(host, 80)
        addrIsIPV6 = False
        for addr in addrInfo:
            # Check if the address resolved to INET6 and the IP address is the
            # same as what was specified (if this is a resolved hostname, then
            # this will not be true.
            if addr[0] == socket.AF_INET6 and addr[4][0] == host:
                addrIsIPV6 = True
                break
        if addrIsIPV6:
            # Add brackets for IPV6 URL compatability.
            host = '[' + host + ']'
        self.url = 'http://%s/%s/' % (host, path.strip('/'))

    def _prepHeaders(self, fp):
        """helper function that sets the appropriate headers
        """
        try:
            self.putrequest('PUT',  self.url+ os.path.basename(fp.name))
            self.putheader('User-Agent', 'isi_gather_info')
            self.putheader('Connection', 'keep-alive')
            self.putheader('Transfer-Encoding', 'chunked')
            self.putheader('Accept', '*/*')
            self.putheader('Expect', '100-continue')
            self.endheaders()
            return (0,'')
        except Exception, e:
            return (1, e)

    def _prepTransfer(self, fp):
        """helper function to get file/time data
        """
        self.fileSize = os.stat(fp.name).st_size
        self.startTime = time.time()
        self.pbar = ProgressBar(self.fileSize)

    def storeFile(self, fp, blocksize=8192):
        """stores the file object (fp) to the host.
            runs the callback function per 'blocksize' data transferred.
        """
        exit_code, error = self._prepHeaders(fp)
        if exit_code:
            return (exit_code, error)
        self._prepTransfer(fp)

        while True:
            buf = fp.read(blocksize)
            if not buf: break
            length = len(buf)
            self.send('%X\r\n' % length)
            self.send(buf + '\r\n')
            self.cbFunc(length)
        self.send('0\r\n\r\n')
        print #flush
        return (0,'')

    #callback func gets uploaded packet size
    def cbFunc(self, bufSize):
        now = time.time()
        self.uploaded += bufSize
        self.timeElapsed = now - self.startTime
        if sys.stdout.isatty() and \
        (now - self.lastTtyTime > self.updateTtyIntVal):
            self.lastTtyTime = now
            self.pbar.update(self.uploaded)

        if self.enableLog and \
        (now - self.lastLogTime > self.updateLogIntVal):
            msg = "Uploaded: %s  out of %s . Time Elapsed: %s" % \
            (format_capacity(self.uploaded), format_capacity(self.fileSize), \
             time.strftime("%H:%M:%S", time.gmtime(self.timeElapsed)))
            self.lastLogTime = now
            self.log.write(msg)


def send_file_to_esrs_via_curl(host, proxy, proxyport, uri, filepath):

    serial_number = isi.app.lib.cluster.DeviceInfo().serial_number()
    guid = arrcfg.config_load().guid
    product_family = "Isilon"
    user_agent = "isi_gather_info"
    custom_headers = (("SerialNumber", serial_number),
                      ("GUID", guid),
                      ("ProductFamily", product_family),
                      )
    program = "/usr/local/bin/curl"
    arguments = []
    arguments.append(program)
    arguments.append("-k")
    arguments.append("-x")
    arguments.append("%s:%s" % (proxy, proxyport))
    arguments.append("--verbose")
    arguments.append("--upload-file")
    arguments.append(filepath)
    for key, value in custom_headers:
        arguments.append("--header")
        arguments.append("%s: %s" % (key, value))
    arguments.append("--user-agent")
    arguments.append(user_agent)
    arguments.append("--progress-bar")
    arguments.append("https://%s%s/%s" %
            (host, uri, os.path.basename(filepath)))

    msg = "Sending using " + " ".join(arguments)
    print msg
    log.write(msg)
    return subprocess.call(arguments)


###<Connectivity>###
class MyFTP(ftplib.FTP):
    """extends ftplib.FTP with a storbinary() that has a callback function.
        with a ProgressBar implementation that prints to stdout.
        current version of Python (2.3) doesn't have callback implemented.
    """
    def __init__(self, host='', user='', passwd='', path='', port='', \
                 proxyhost='', proxyport=''):
        global log
        self.log = log
        self.startTime  = 0
        self.fileSize   = 0
        self.uploaded   = 0
        self.enableLog  = True
        self.updateLogIntVal = 3   #interval to log to file
        self.updateTtyIntVal = 1   #interval to update progress
        self.lastLogTime = 0
        self.lastTtyTime = 0
        self.pbar = ''             #ProgressBar object

        ftplib.FTP.__init__(self)
        try:
            if proxyhost:
                print "connecting via proxy: %s " % proxyhost
                # to ftp to a proxy, we connect to the proxy, as uerser
                if(proxyport):
                    self.connect(proxyhost, proxyport)
                else:
                    self.connect(proxyhost)
            else:
                self.connect(host, port)
        except Exception, e:
            err_msg = "FTP Error: Could not connect to %s on port %s" % \
                      ((proxyhost or host), (proxyport or port))
            print e
            print err_msg
            self.log.write(err_msg)
            raise
        try:
            if proxyhost:
                #we login to the proxy with user@non_proxy_host
                #XXX this does not work with other proxy types: use urllib2
                self.login('%s@%s:%s' % (user, host, port), passwd)
            else:
                self.login(user, passwd)
        except Exception, e:
            print e
            err_msg = "FTP Error: Could not authenticate to %s with user %s " % \
                      (host, user)
            print err_msg
            self.log.write(err_msg)
            raise
        try:
            self.cwd(path)
        except Exception, e:
            print e

            print "unable to change FTP dir to %s attempting uploading to %s " \
                  % (path, self.pwd())



    def storbinary(self, cmd, fp, blocksize=8192):
        """Store a file in binary mode with callback function """

        self.voidcmd('TYPE I')
        conn = self.transfercmd(cmd)
        self._prepTransfer(fp)

        while 1:
            buf = fp.read(blocksize)
            if not buf: break
            conn.sendall(buf)
            self.cbFunc(len(buf))
        conn.close()
        return self.voidresp()

    def _prepTransfer(self, fp):
        self.fileSize = os.stat(fp.name).st_size    #set filesize
        self.startTime = time.time()                #set Starttime
        self.pbar = ProgressBar(self.fileSize)

    #callback func gets uploaded packet size
    def cbFunc(self, bufSize):
        now = time.time()
        self.uploaded += bufSize
        self.timeElapsed = now - self.startTime
        if sys.stdout.isatty() and \
        (now - self.lastTtyTime > self.updateTtyIntVal):
            self.pbar.update(self.uploaded)
            self.lastTtyTime = now

        #write to gather file
        if self.enableLog and \
        (now - self.lastLogTime > self.updateLogIntVal):
            msg = "Uploaded: %s  out of %s . Time Elapsed: %s" % \
            (format_capacity(self.uploaded), format_capacity(self.fileSize), \
             time.strftime("%H:%M:%S", time.gmtime(self.timeElapsed)))
            self.lastLogTime = now
            self.log.write(msg)
###</Connectivity>###

####<XML CONFIG>###
class MXDconfig:
    """this class allows for reading/writing to the xml config file in
    /etc/mcp/ (sys &|| override)/gather.xml"""
    def __init__(self, filename='gather.xml'):
        from isi.config import mxd
        import shutil

        self.filename = filename
        self.fname = CONFIGDIR + 'sys/' + filename
        self.deltafname = CONFIGDIR + 'override/' + filename
        self.rules = {}
        self.tmpdir = '/tmp/'    #have to use /tmp because TMP_SUPPORT_DIR is not defined yet

        try:
            self.doc = mxd.loadmxd(self.fname, self.deltafname)
            self.root = self.doc.getroot()
        except mxd.error, e:
            if e[0] == 22: #override/gather.xml is corrupt
                #remove old /override config and rewrite new one
                print "Error. Corrupt config file "\
                "(/etc/mcp/override/gather.xml)"
                print "Resetting config file."
                tempConfigFilePath = self.tmpdir + self.filename
                try:
                    self.doc = mxd.loadmxd(self.fname)
                    self.root = self.doc.getroot()
                    self.doc.save(tempConfigFilePath)
                    shutil.move(tempConfigFilePath, self.deltafname)
                except mxd.error, e:
                    print "Reset Failed: Using given "\
                    "command line options"
                    pass

    def getconfig(self):
        """parses xml files and returns a dict"""
        try:
            if self.root != None:
                #gathers the 'settings' node of the xml file and makes a \
                #dictionary
                for each in self.root.getchildren():
                    if len(each.gettext()) > 0:
                        self.rules[each.getattributes()['name']] = \
                                each.gettext()
            return self.rules
        except Exception, e:
            print "Error with config file. %s" % str(e)

    def setconfig(self, rules):
        """takes a dict of rules and saves them"""
        if rules != {}:
            for key in rules.keys():
                for each in self.root.getchildren():
                    if each.getattributes()['name'] == key:
                        each.settext(str(rules.get(key)))
            tempConfigFilePath = self.tmpdir + self.filename
            self.doc.save(tempConfigFilePath)                       #save the file in tmp directory
            shutil.move(tempConfigFilePath, self.deltafname)        #move it to config directory.

def saveConfigOptions():
    """compiles the settings and saves them to the xml config files at
     /etc/mcp/(sys &| override)/gather.xml"""
    global options
    rules = {}  #preps a dict of options to write to config xml
    #these are the options from the cmd line to be saved to xml.
    #the items correspond to keys in "OPTIONS" dict
    optionsToSave = ['incremental', 'ftp_host', 'ftp_port', 'ftp_path', 'ftp_user',
                     'ftp_proxy_host', 'ftp_proxy_port', 'ftp_pass', 'http_host', 'http_path',
                     'ftp_enable', 'http_enable',
                     'upload_package']
    for each in optionsToSave:
        try:
            if options.__dict__[each]:
                rules.update({each : options.__dict__[each]})
        except KeyError:
            pass
        except Exception, e:     #allow for clean failures
            print "Error saving config: %s" % str(e)

    #take care of non string config settings
    rules.update({'incremental':    options.__dict__['incremental']})
    rules.update({'upload_package': options.__dict__['upload_package']})
    rules.update({'http_enable': options.__dict__['http_enable']})
    rules.update({'ftp_enable': options.__dict__['ftp_enable']})

    config = MXDconfig()
    config.setconfig(rules)
    return


def loadConfigOptions():
    """loads the xml config options into the OPTIONS object"""
    global options

    try:
        config = MXDconfig()
        rules = config.getconfig()
        options.__dict__.update(rules)
    except Exception, e:
        print "Error loading config options from file. Using cmdline options"

    # take care of non-string settings. Incremental is a bool
    # It is possible that these values accidentally become None.
    # Then, use default value.
    try:
        options.__dict__.update( {'incremental'   : \
                                  (rules['incremental'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'incremental': False} )
    try:
        options.__dict__.update( {'upload_package': \
                                  (rules['upload_package'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'upload_package': True} )
    try:
        options.__dict__.update( {'ftp_enable': \
                                  (rules['ftp_enable'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'ftp_enable': True} )
    try:
        options.__dict__.update( {'http_enable': \
                                  (rules['http_enable'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'http_enable': True} )

    return

####</XML CONFIG>###

######<DEBUG/ PROGRESS INDICATORS>##########
class Dots(threading.Thread):
    """spawns a thread to prints dots to tty to indicate process activity"""
    def __init__(self, name="dots"):
        self._stopevent = threading.Event()
        self.sleeptime = 0.2
        threading.Thread.__init__(self,name=name)

    def run(self):
        while not self._stopevent.isSet() and sys.stdout.isatty():
            sys.stdout.write('.')
            sys.stdout.flush()
            time.sleep(self.sleeptime)
        if not sys.stdout.isatty():    #no need to exist anymore
            self._stopevent.set()

    def join(self, timeout=None):
        print "" #newline
        if (self.isAlive()):
            self._stopevent.set()
            threading.Thread.join(self, timeout)

class ProgressBar(object):
    """Creates a Progress indicator in a bar fashion on the tty terminal window.
        progress is updated as update is called (usually after "blocksize" of
        data is transferred)
    """
    def __init__(self, total):
        self.done               = 0
        self.ETA                = 0
        self.timeElapsed        = 0
        self.percentComplete    = 0
        self.startTime          = time.time()
        self.total              = total
        if sys.stdout.isatty():
            try:                        #import libs for determining tty width
                from fcntl import ioctl
                import termios
                #from array import array
            except ImportError:
                print "import error in fnctl and termios. screen width cannot " \
                 "be determined"
                pass

            try:
                self.screenWidth = array('h', \
                                         ioctl(sys.stdout,termios.TIOCGWINSZ,\
                                               '\0'*8))[:2][1]
            except:
                self.screenWidth = 80

    def update(self, done):
        """calculates and updates the progress.
        """
        self.done = done
        self.percentComplete = float(self.done)/float(self.total)*100
        self.timeElapsed = time.time() - self.startTime
        try:
            self.ETA = (100.0/self.percentComplete - 1)*self.timeElapsed
        except ZeroDivisionError, e:        #so it doesn't freakout if ETA
            self.ETA = 0
        sys.stdout.write( self.printer() )
        sys.stdout.flush()
        if self.percentComplete == 100:
            print

    def formatTime(self, timeX):
        """helper time function.
        Caveat: results of time functions vary between FreeBSD and Linux
        """
        return time.strftime("%H:%M:%S",time.gmtime(timeX))

    def printer(self):
        """creates the text string to be displayed. ex:
        90.00% [#################################---] ETA: 00:00:01 Elapsed: 00:00:03 2880/3200 KB
        """
        lhs = "%.2f%% [" % self.percentComplete
        rhs = "]"
        rhs += " ETA: " + self.formatTime(self.ETA)
        rhs += " Elapsed: " + self.formatTime(self.timeElapsed)
        rhs += " %s/%s " %( format_capacity(self.done),
                                format_capacity(self.total))
        #lspace = space to left justify
        lspace = self.screenWidth - len(str(lhs)) - len(str(rhs))
        bar = "#"*int((self.percentComplete/100.0)*lspace)
        bar = bar.ljust(lspace).replace(" ","-")
        out = lhs + bar + rhs + '\r'
        return out
##########</DEBUG/ PROGRESS INDICATORS>##########

class MyLock(object):
    """class to handle .lock files
        """
    def __init__(self, path = GATHER_STATUS_LOCK, verbose = False):
        self.lockFD = ''
        self.lockfile = path
        self.pid = os.getpid()
        self.verbose = verbose

    def isPidRunning(self, pid):
        """checks if a given isi_gather_info is an active process"""
        cmd = """ps -o command= -p %d """
        try:
            (exit_code, output) = procs.get_cmd_output(cmd % pid)
            if exit_code:
                return False
            if output and len(output):
                #if we get an output, it'll be just one element -> pid
                return 'isi_gather_info' in output[0]
            return False
        except:
            return False

    def _readlock(self):
        """reads the contents of a lock file. ie pid of the owner"""
        try:
            self.lockFD=os.open(self.lockfile, os.O_RDONLY)
            pid = os.read(self.lockFD, 100)
            os.close(self.lockFD)
            if pid != '':
                pid = int(pid)
            else:
                pid = 10**8
            return pid
        except:
            return False

    def _lock(self):
        """helper to lock()"""
        try:
            self.lockFD = os.open(self.lockfile, \
                    os.O_EXCL | os.O_CREAT | os.O_RDWR, 0400)
            os.write(self.lockFD, str(self.pid))
            os.close(self.lockFD)
            return True
        except OSError,e:
            if e.errno == 17:   #file exists!
                return False
            return False
        except:                 #any others
            return False

    def ownlock(self):
        """do i own lock"""
        try:
            lock = self._readlock()
            if lock:
                return self.pid == lock
            return False
        except:
            return False

    def lock(self):
        """locks. writes own pid to lock file"""

        #is gather-status unlocked? try locking
        if not os.path.exists(self.lockfile):
            return self._lock()
        else:
            pid = self._readlock()
            #is gather-status.lock owner running? ie lock expired?
            if not self.isPidRunning(pid):
                print "Unlocking gather-status"
                self.unlock()
                return self._lock()
            else:
                print "%s is currently locked. Status Updates will not be written to file." \
                % (self.lockfile)
        return False

    def unlock(self):
        """ unlocks lockfile """
        if os.path.exists(self.lockfile):
            print "Gather-status unlocked"
            try:
                os.remove(self.lockfile)
            except Exception, e:
                print "Error unlocking lockfile"
                return False
            return True
        else:
            print ("Gather-status lock does not exit")
            return False

class Logger(threading.Thread):
    """runs logging to file as a thread. timestamps the log file every
        self.updateInterval seconds. also allows async writes, as used
        to update program status.
    """
    def __init__(self, name="Logger", filename = GATHER_STATUS_PATH):
        self._stopevent = threading.Event()
        self.updateInterval = 3
        self.lastLogTime = 0
        self.fileFD = sys.stdout
        self.filename = filename
        self.disable = False
        self.flock = MyLock()

        self.flock.owner = self.flock.lock()
        if self.flock.owner:
            try:
                self.fileFD = open(self.filename, 'w')
            except IOError, e:
                print "Could not open gather-status file. No isi status will "\
                "be stored."
                self.disable = True
        else:
            self.disable = True
        threading.Thread.__init__(self, name=name)
        self.start()

    def run(self):
        """timestamps the file as long as the thread is active"""
        while (not self._stopevent.isSet()) and (not self.disable):
            self.write()
            time.sleep(self.updateInterval)

    def write(self, msg="", priority=False):
        """priority write. update with msg"""
        if (not self._stopevent.isSet()) and (not self.disable):
            try:
                self.lastLogTime = time.time()
                timestamp = "[%s] " % time.strftime("%Y-%m-%d %H:%M:%S")
                self.fileFD.writelines(timestamp + msg + '\n')
                self.fileFD.flush()
            except IOError, e:
                #this is to escape any errors generated by a closed terminal
                self.join(None)
                pass

    def join(self, timeout=None):
        self._stopevent.set()
        if (self.flock.owner and self.flock.ownlock()):
            try:
                threading.Thread.join(self, timeout)
                self.fileFD.close()
                self.flock.unlock()
            except Exception,e:
                print "Error unlocking gather log"
                pass

def extendCLOptionsDefault(option, defaultParam):
    """extends the default optional parameter of a cmdline option. mainly
       useful for using with getopts.
    ex: isi_gather_info --incremental becomes isi_gather_info --incremental on
    """
    try:
        if (option in sys.argv[1:]):
            #check to see its not the last one
            if sys.argv.index(option) != len(sys.argv)-1:
                #check to see if following text is option
                if sys.argv[sys.argv.index(option)+1].startswith('-'):
                    sys.argv.insert(
                            sys.argv.index(option)+1,
                            defaultParam)
            else:
                sys.argv.insert(
                        sys.argv.index(option)+1,
                        defaultParam)
    except Exception,e:
        print "Error in parsing non argument option. %s" % str(e)

def get_options():
    global cluster, options, log, re_upload

    ##parse "--incremental" option as "--incremental on"
    extendCLOptionsDefault('--incremental', 'on')

    try:
        opts, args = getopt.getopt(sys.argv[1:],
            'hvlu:p:f:i:x:n:1:s:S:a:A:t:T:IL',
            ['help', 'version', 'list', 'user=', 'password=',
             'include=', 'nodes=', 'tarfile=', 'temp=', 'ifs', 'incremental=',
             'nologs', 'debug', 'verbose', 'varlog_all', 'varlog_recent',
             'upload', 'noupload', 'ftp', 'noftp', 'ftp-user=', 'ftp-pass=',
             'ftp-host=', 'ftp-path=', 'ftp-mode=', 'skip-node-check', 'save',
             'save-only', 'no-config', 'http', 'nohttp', 'http-host=',
             'http-path=', 'verify-upload', 'http-proxy=', 'http-proxy-port=',
             'ftp-port=', 'ftp-proxy=', 'ftp-proxy-port=',
             're-upload=', 'skip-size-check',
             'group=', 'incremental=', 'noconfig', 'tardir=',
             'symlinkdir=', 'local-only', 'esrs', 'noesrs', 'no-dumps',
             'clean-cores', 'clean-dumps',
             'clean-all', 'no-cores', 'dumps', 'cores', 'upgrade-archive'
     ])

        # load saved options unless --noconfig is specified
        if '--noconfig' not in [i[0] for i in opts]:
            loadConfigOptions()

        for o, a in opts:
            if o in ['-h', '--help']:
                usage(0)
            elif o in ['-v', '--version']:
                version()
            elif o in ['-l', '--list']:
                listUtils()
                listGroups()
                sys.exit(0)
            elif o in ['--noconfig']:
                pass     # nothing to do here; handled above
            elif o in ['-u', '--user']:
                cluster.user = a
            elif o in ['-p', '--password']:
                cluster.password = a
            elif o in ['-f']:
                options.extra_file_list.extend(a.split(','))
            elif o in ['-i', '--include']:
                options.includes.extend(a.split(','))
            elif o in ['-x']:
                for each in a.split(','):
                    try:
                        EXEMPT.remove(each)
                    except ValueError, e:
                        error("Utility %s does not exist to be excluded." % \
                              (each))
            elif o in ['-n', '--nodes']:
                options.isi_nodes_args += [o, a]
            elif o in ['--local-only']:
                options.local_only = True
            elif o in ['--skip-node-check']:
                options.skip_node_check = True
            elif o in ['-1']:
                if not is_compliance():
                    options.gather_expr_local.extend(a.split(','))
                else:
                    error('gather_exprs are not supported in ' + \
                            'compliance mode.')
            elif o in ['-s']:
                if not is_compliance():
                    options.gather_script.extend(a.split(','))
                else:
                    error('gather scripts are not supported in ' + \
                            'compliance mode.')
            elif o in ['-S']:
                if not is_compliance():
                    options.gather_expr.extend(a.split(','))
                else:
                    error('gather_exprs are not supported in ' + \
                            'compliance mode.')
            elif o in ['-a']:
                if not is_compliance():
                    options.analysis_script.extend(a.split(','))
                else:
                    error('analysis scripts are not supported in ' + \
                            'compliance mode.')
                options.analysis_script.extend(a.split(','))
            elif o in ['-A']:
                if not is_compliance():
                    options.analysis_expr.extend(a.split(','))
                else:
                    error('analysis_exprs are not supported in ' + \
                            'compliance mode.')
                options.analysis_expr.extend(a.split(','))
            elif o in ['-t', '--tarfile']:
                options.tarfile = a
            elif o in ['--tardir']:
                # this is a lot like -T, except -T still creates a 'pkg'
                # directory underneath the specified directory, which is
                # not always desirable.  I'd replace -T but I have no
                # idea if its being used.
                if not a.startswith('/'):
                    print "Tar directory %s invalid; it must begin with /"
                    sys.exit(1)

                options.tardir = a
            elif o in ['--symlinkdir']:
                if not a.startswith('/'):
                    print "Symlink dir %s invalid; it must begin with /"
                    sys.exit(1)

                options.symlinkdir = a
            elif o in ['-T', '--temp']:
                options.tempdir = a.lstrip()
                if os.path.abspath(options.tempdir) in ('/ifs', '/ifs/data',
                        '/ifs/.ifsvar'):
                    raise ValueError("Invalid path for tempdir specified. " + \
                            "I'm very sure you don't actually want to do this.")
            elif o in ['-I', '--ifs']:
                options.use_ifs = True
            elif o in ['-L', '--local-storage']:
                options.use_ifs = False
            elif (o in ['--incremental']):
                if a == 'off':
                    options.incremental = False
                else:
                    options.incremental = True
            elif o in ['--nologs']:
                options.no_exempt_logs = True
            elif o in ['--debug']:
                options.debug = True
            elif o in ['--verbose']:
                options.verbose = True
            elif o in ['--varlog_all']:
                # This is the default
                EXEMPT[0] = 'varlog_all'
            elif o in ['--varlog_recent']:
                EXEMPT[0] = 'varlog_recent'
            elif o in ['--noupload']:
                options.upload_package = False
            elif o in ['--upload']:
                options.upload_package = True
            elif o in ['--noftp']:
                options.ftp_enable = False
            elif o in ['--ftp']:
                options.ftp_enable = True
            elif o in ['--ftp-user']:
                options.ftp_user = a
            elif o in ['--ftp-pass']:
                options.ftp_pass = a
            elif o in ['--ftp-host']:
                options.ftp_host = a
            elif o in ['--ftp-path']:
                options.ftp_path = a
            elif o in ['--ftp-port']:
                options.ftp_port = a
            elif o in ['--ftp-mode']:
                if a == "active":
                    options.try_active_ftp = True
                    options.try_passive_ftp = False
                elif a == "passive":
                    options.try_active_ftp = False
                    options.try_passive_ftp = True
                elif a == "both":
                    options.try_active_ftp = True
                    options.try_passive_ftp = True
                else:
                    sys.stderr.write("Valid options for --ftp-mode are: "\
                                     "both, active and passive")
                    usage(1)
            elif o in ['--ftp-proxy']:
                options.ftp_proxy_host = a
            elif o in ['--skip-size-check']:
                options.skip_size_check = True
            elif o in ['--ftp-proxy-port']:
                options.ftp_proxy_port = a
            elif o in ['--nohttp']:
                options.http_enable = False
            elif o in ['--http']:
                options.http_enable = True
            elif o in ['--http-host']:
                options.http_host = a
            elif o in ['--http-path']:
                options.http_path = a
            elif o in ['--http-proxy']:
                options.http_proxy = a
            elif o in ['--http-proxy-port']:
                options.http_proxy_port = a
            elif o in ['--esrs']:
                options.esrs = True
            elif o in ['--noesrs']:
                options.esrs = False
            elif o in ['--cores']:
                options.cores = True
            elif o in ['--dumps']:
                options.dumps = True
            elif o in ['--clean-cores']:
                options.clean_cores = True
            elif o in ['--clean-dumps']:
                options.clean_dumps = True
            elif o in ['--clean-all']:
                options.clean_cores = True
                options.clean_dumps = True
            elif o in ['--no-dumps']:
                try:
                    EXEMPT.remove('isi_hangdump')
                except ValueError, e:
                    error("isi_hangdump does not exist to be excluded.")
            elif o in ['--no-cores']:
                try:
                    EXEMPT.remove('cores')
                    EXEMPT.remove('minidumps')
                except ValueError, e:
                    error("cores or minidumps does not exist to be excluded.")
            elif o in ['--verify-upload']:
                options.verify_upload = True
            elif o in ['--save']:
                saveConfigOptions()
            elif o in ['--save-only']:
                options.save_only = True
            elif o in ['--re-upload']:
                options.re_upload = True
                re_upload = a
            elif o in ['--group']:
                try:
                    options.group_utils.extend(GROUPS[a])
                    options.group.append(a)
                except KeyError, e:
                    print "Invalid group specified: %s" % a
                    listGroups()
                    sys.exit(1)
            elif o in ['--upgrade-archive']:
                options.upgrade_archive = True
            else:
                print "Unknown argument: %s" % o
                usage(1)

    except getopt.GetoptError, e:
        error(str(e.msg))

    if is_compliance():
        cluster.user = 'compadmin'

def usage(exitCode):
    sys.stderr.write('''
Usage: %s [OPTION]...

Options:
  -h                  Print this message and exit.
  -v                  Print version info and exit.
  -u USER             Login as USER instead of default root.
  -p PASSWORD         Use PASSWORD.
  -i                  Include only the listed utility.  See also the
                      -l option for a list of utilities to include.
                      The special value "all" may be used to include
                      every known utility.
  --incremental       Gather only logs changed since last log upload
  -l                  List utilities and groups which can be included
                      (see -i and --group)
  -f FILENAME         Gather FILENAME from each node. The value must be an absolute path.
  -n NODES            Gather information only from the given nodes.
                      Must be a list or range of LNNs (what you see
                      in "isi stat" output). For example: '1,4-10,12,14'.
                      If no nodes are explicitly listed, the whole array
                      is used.  Note that nodes are automatically
                      excluded if they are down.
  --local-only        Do not gather any information from other nodes.
  --skip-node-check   Skip check for node availability.
  -s gather_script    *Run gather_script on every node.
  -S gather_expr      *Run gather_expr on every node.
  -1 gather_expr      *Run gather_expr on the local node.
  -a analysis_script  *Run analysis_script on results.
  -A analysis_expr    *Run analysis_expr on every node.
  -t TARFILE          Save all results to TARFILE instead of default tar file.
  -x exclude_tool     Excludes the specified tool from being gathered from each node.
  -I                  Save results to IFS (default)
  -L                  Save all results to local storage: %s
  -T TEMPDIR          Save all results to TEMPDIR instead of default dir.
                      (overrides -L and -I)
  --tardir <dir>      Place the final package directly into <dir>.
  --symlinkdir <dir>  Create a symlink to the final package in <dir>.
  --varlog_recent     Gather all logs in /var/log, but not the compressed and
                      rotated old logs (Default is for all logs).
  --varlog_all        Gather all logs in /var/log, including compressed and
                      rotated old logs (Default).
  --nologs            Do not gather normal required minimal logs.
  --group <name>      Add a specific group of utilities to the tarball.
  --clean-cores       Delete cores from /var/crash after successful compression of the package.
  --clean-dumps       Delete dumps from /var/crash after successful compression of the package.
  --clean-all         Delete cores and dumps from /var/crash after successful compression of the package.
  --no-dumps          Don't get hangdumps.
  --dumps             Add dumps to the package.
  --no-cores          Don't get cores or minidumps.
  --cores             Add cores to the package.
  --upgrade-archive   Add upgrade archive to the package.
  --debug             Display debugging messages.
  --verbose           Display more detailed information.

Configuration:
  --noconfig         Use built-in defaults and bypass the configuration file.
  --save-only        Save the CLI specified configuration to file and exit.
  --save             Save the CLI specified configuration to file and run.

Upload Options:
  --upload            Upload logs to isilon automatically. (Default)
  --noupload          Do not upload logs to isilon automatically.
  --re-upload FILE    Re-uploads the specified FILE.
  --verify-upload     Creates a tar file and uploads to test connectivity.

  HTTP Upload Options:
    --http                 Attempt HTTP upload. (Default)
    --nohttp               Do not attempt HTTP upload.
    --http-host HOST       Specifies alternate HTTP site for upload.
    --http-path DIR        Specifies alternate HTTP upload directory.
    --http-proxy HOST      Specifies Proxy server to use.
    --http-proxy-port PORT Specifies Proxy port to use.

  FTP Upload Options:
    --ftp                 Attempt FTP upload. (Default)
    --noftp               Do not attempt FTP upload.
    --ftp-user USER       Specify alternate user for FTP.  (Default: anonymous)
    --ftp-pass PASS       Specify alternate password for FTP.
    --ftp-host HOST       Specifies alternate FTP site for upload.
    --ftp-path DIR        Specifies alternate FTP upload directory.
    --ftp-port PORT       Specifies alternate FTP port for upload.
    --ftp-proxy HOST      Specifies Proxy server to use.
    --ftp-proxy-port PORT Specifies Proxy port to use.
    --ftp-mode MODE       Mode of FTP file transfer (default: attempt both)
                          valid: both, active, passive

  ESRS Upload Options:
    --esrs                Attempt ESRS upload. (Default)
    --noesrs              Do not attempt ESRS upload.

  Multiple instances of -i, -f, -s, -S, -1 are allowed.
  gather_expr, analysis_expr can be quoted.
  Default temporary directory is %s
  (change with -L or -T)

  * Not available on compliance mode.

''' % (PROGNAME, VAR_SUPPORT_DIR, IFS_SUPPORT_DIR))
    sys.exit(exitCode)

def version():
    print '%s version: %s' % (PROGNAME, REVISION)
    sys.exit(0)

if __name__ == '__main__':
    main()
