#!/usr/bin/python
# ex: set shiftwidth=4 tabstop=4 expandtab:

# This version has much work on extending isi_gather_info to handle generic
# info,file,script output -- Indeed, anything cluster-wide should be possible
# if it can be encapsulated into a provided node-run script.
#
# Be sure to maintain backwards compatibility with Releases 3.0, 3.5, 4.0.
# In absence of new options, all behavior should be identical to previous
# isi_gather_info versions.
#
# This script will use /ifs by default to store results.  This is a
# change from previous behavior and was requested explicitly from
# support.
#
# Selected Options Info:
#  o Filenames:
#    Gathered just like Required Logs: Will handle both Dirs and Files.
#  o gather_script, gather_expr:
#    Will be added to the generated Bash script with UTILITIES.
#    Output will be piped to an appropriate named file, based on generated
#    command.
#  o analysis_script:
#    Will be executed before tarball generation.
#    Output will be piped to an appropriate named file.
#  o tarfile:
#    Replaces the auto-generated pkg_path_name.
#    If full path specified, use that. Else, use pkg_path_name code for path.
#  o nologs:
#    Only way to skip gathering required logs.

import errno
import ftplib
import getopt
import getpass
import os
import random
import re
import signal
import string
import sys
import tempfile
import time
import threading
import thread
import traceback
import httplib
import subprocess

import socket
import isi.config.array as array
import isi.config.arrcfg as arrcfg
import isi.net.rpc as rpc
import isi.gconfig as gconfig

import isi.app.lib.cluster
import isi.app.lib.procs as procs
import isi.fs.restripe.group as group
from isi.config import mxd
from isi.ui.lib.guiformat import format_capacity
from isi.app.lib.emailer import Emailer, EmailAttachmentFromFile, EmailError
from isi.misc import sysctlbyname

import shutil

# for package xml metadata
import xml.etree.ElementTree as etree
import isi.celog.attachments
import isi.celog.celog_config_api
import isi.config.arrcfg

PROGNAME = os.path.basename(sys.argv[0])
# Bug 167340 - switch to git attr ident
REVISION = '$Id: 90c1cb9f0ddc3fa3419c57479d90a1e4a6c8c8f9 $'

EMAIL_REGEX = re.compile('[^@]+@[^\.]+(\.[^\.]+)+')

try:
    True
except NameError:
    True  = 1
    False = 0

SUPPORT_DIR           = None
TMP_SUPPORT_DIR       = None
LOCAL_TMP_SUPPORT_DIR = None
PKG_SUPPORT_DIR       = None
RUN_DIR               = None
GATHER_SCRIPT         = None
ISI_NODES_OUTPUT      = None
ISI_FIRMWARE_OUTPUT   = None
PACKAGE_INFO          = None
CELOG_EVENTS_INFO     = None
SAVED_COMMAND_LINE    = None
GATHER_RUN_FILE       = None
IFS_READONLY          = False

_TMP_SUPPORT_DIR       = 'tmp/'
_PKG_SUPPORT_DIR       = 'pkg/'
_GATHER_SCRIPT         = '_gather'
_LOCAL_GATHER_SCRIPT   = '_local_gather'
_LOCAL_TMP_SUPPORT_DIR = "local"
_ISI_NODES_OUTPUT      = 'nodes_info'
_ISI_FIRMWARE_OUTPUT   = 'firmware'
_PACKAGE_INFO          = 'package_info.xml'
_CELOG_EVENTS_INFO     = 'celog_events.xml'
_SAVED_COMMAND_LINE    = 'command_line'
_INCREMENTAL_FILE      = 'last_gather'
_FULLGATHER_FILE       = 'last_full_gather'

VAR_SUPPORT_DIR = '/var/crash/support/'
IFS_SUPPORT_DIR = '/ifs/data/Isilon_Support/'
RUN_SUPPORT_DIR = os.path.join(os.path.sep + 'ifs', '.ifsvar',
        'run' + os.path.sep)
RUN_VAR_SUPPORT_DIR = os.path.join(VAR_SUPPORT_DIR, 'run'
         + os.path.sep + 'tmp' + os.path.sep)

GATHER_STATUS_PATH  = '/ifs/.ifsvar/run/gather-status'
GATHER_STATUS_LOCK  = GATHER_STATUS_PATH + '.lock'

CONFIGDIR = '/etc/mcp/'

# 'exe': cmd, name: cmd==command, name==output filename
# 'tar': cmd, name: cmd==cd to parent dir; name==tarball_filename tarball_dir
# See generate_scripts for the consumer of this information.
UTILITIES = {
    'abr_config' : ('exe', "${SUDO}/usr/bin/isi_make_abr &>/dev/null; "
                                "${SUDO}/usr/bin/isi_inventory_tool "
                                "--display --itemType=asbuilt", "asbuilt.xml"),
    'aspera_config' : ('local_tar', "cd /", "aspera_config.tar " \
                                 "/ifs/.ifsvar/aspera/*"),
    'atacontrol'      : ('exe', "${SUDO}/sbin/atacontrol list", 'atacontrol'),
    'celog_node_db'   : ('tar', "cd /", "celog_db.tar /var/db/celog/*"),
    'celog_ifsvar_db' : ('local_tar', "cd /", "celog_ifsvar_db.tar " \
                                "/ifs/.ifsvar/db/celog/*"),
    'configs'         : ('tar', "cd /", "local_configs.tar "    \
                                "etc/mcp "                  \
                                "etc/isilon_serial_number " \
                                "etc/isilon_system_config "),
    'cto_summary'     : ('exe', "${SUDO}/usr/bin/isi_cto_summary",
                                "cto_summary"),
    'dedupe_reports'  : ('local_exe', "/usr/bin/isi dedupe reports list -v",
                                      "dedupe_reports"),
    'dedupe_settings' : ('local_exe', "/usr/bin/isi dedupe settings view",
                                      "dedupe_settings"),
    'dedupe_stats'    : ('local_exe', "/usr/bin/isi dedupe stats",
                                      "dedupe_stats"),
    'compliance'      : ('local_exe', "/sbin/sysctl isi.compliance", "isi.compliance"),
    'df'              : ('exe', "/bin/df -li", "df"),
    'du'              : ('exe', "/usr/bin/du -hd3 /var", "du"),
    'etcifs'          : ('tar', "cd /etc", "etcifs.tar ifs"),
    'etc'             : ('tar', "cd /", "etc.tar etc"),
    'efs.gmp.group'   : ('exe', '/sbin/sysctl efs.gmp.group', 'efs.gmp.group'),
    'efs.l3'          : ('exe', '/sbin/sysctl efs.l3', 'efs.l3'),
    'efs.lbm.drive_space':
                        ('exe', '/sbin/sysctl efs.lbm.drive_space',
                                'efs.lbm.drive_space'),
    'efs.snapshot'    : ('exe', '/sbin/sysctl efs.snapshot', 'efs.snapshot'),
    'efs.stf'         : ('exe', '/sbin/sysctl efs.stf', 'efs.stf'),
    'fstat'           : ('exe', "/usr/bin/fstat", "fstat"),
    'gconfig_local'   : ('tar', "cd /", "gconfig_local.tar " \
                                "/var/ifs/*.gc " \
                                "/etc/ifs/*.gc"),
    'gconfig_ifsvar'  : ('local_tar', "cd /", "gconfig_ifsvar.tar " \
                                "/ifs/.ifsvar/main_config*" ),
    'history'         : ('tar', "cd /", "history.tar /var/ifs/history" ),
    'zsh_history'     : ('tar', "cd /root", "zsh_history.tar .zsh_history" ),
    'zsh_history_compadmin'  : ('tar', "cd /compadmin", "zsh_history_compadmin.tar .zsh_history" ),
    'hw.ata'          : ('exe', "/sbin/sysctl hw.ata", "hw.ata"),
    'hw.disk_event'   : ('exe', "/sbin/sysctl hw.disk_event", "hw.disk_event"),
    'hw.ecclist'      : ('exe', "/sbin/sysctl hw.ecclist", "hw.ecclist"),
    'hw.iosched'       : ('exe', "/sbin/sysctl hw.iosched", "hw.iosched"),
    'ifconfig'        : ('exe', "/sbin/ifconfig -a", "ifconfig"),
    'ifsvar_db_pkg'   : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_db_pkg.tar "\
                                      "--exclude '*.tar' db/pkg"),
    'ifsvar_db_pkg_tars': ('local_exe',
                            "/usr/bin/find /ifs/.ifsvar/db/pkg -name '*.tar' "\
                            "-ls -exec /usr/bin/tar -tvf {} \;",
                            "ifsvar_db_pkg_tars"),
    'ifsvar_etc'      : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_etc.tar etc"),
    'ifsvar_modules'  : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_modules.tar "
                         "--exclude stats --exclude bedrs/dedupe_metadata "
                         "--exclude bedrs/dry_run_metadata "
                         "--exclude 'fsa/pub' --exclude 'fsa/work' "
                         "--exclude 'jobengine/cp' modules"),
    'ifsvar_modules_stats'  : ('local_tar', "cd /ifs/.ifsvar", \
                               "ifsvar_modules_stats.tar " \
                                "modules/stats"),
    'ifsvar_modules_fsa'  : ('local_tar', "cd /ifs/.ifsvar", \
                               "ifsvar_modules_fsa.tar " \
                                "modules/fsa/pub"),
    'ifsvar_modules_jobengine_cp' : ('local_tar', "cd /ifs/.ifsvar/modules/jobengine", \
                               "ifsvar_modules_jobengine_cp.tar " \
                                "cp"),
    'ifsvar_tmp'      : ('local_tar', "cd /ifs/.ifsvar", "ifsvar_tmp.tar tmp" ),
    'isi_alerts'      : ('local_exe', "${SUDO}/usr/bin/isi events list -w", "isi_alerts"),
    # this utility takes long time.
    'isi_bug_info'    : ('exe', "${SUDO}/usr/bin/isi_bug_info", "isi_bug_info"),
    'isi_devices'     : ('exe', "${SUDO}/usr/bin/isi devices", "isi_devices"),
    'isi_devices_fwstatus': ('exe', "${SUDO}/usr/bin/isi devices -a fwstatus", "isi_devices_fwstatus"),
    'isi_drivefirmware': ('local_exe', "${SUDO}/usr/bin/isi drivefirmware status", "isi_drivefirmware"),
    'isi_drivenum'    : ('exe', "${SUDO}/usr/bin/isi_drivenum -A", "isi_drivenum"),
    'isi_disk_inject' : ('exe', '${SUDO}/usr/bin/isi_hwtools/isi_disk_inject -l',
                                'isi_disk_inject'),
    'isi_diskpool'    : ('local_exe', "/usr/bin/disi -I diskpool list -v",
                                "isi_diskpool"),
    'efs.bam.disk_pool_db':
                        ('exe', "/sbin/sysctl efs.bam.disk_pool_db",
                                    'efs.bam.disk_pool_db'),
    'isi_statistics_drive'  : ('exe', "${SUDO}/usr/bin/isi statistics drive " \
				"--degraded", "isi_statistics_drive"),
    'isi_iscsi_target': ('local_exe', "/usr/bin/isi target ls -l -v",
                                "isi_iscsi_target"),
    'ifsvar_iscsi'    : ('local_tar', "cd /ifs/.ifsvar",
                                "ifsvar_iscsi.tar iscsi"),
    'isi_dmi_log'     : ('exe', "${SUDO}/usr/bin/isi_gather_memory_ecc_info", "isi_dmilog"),
    'isi_hw_status'   : ('exe', "${SUDO}/usr/bin/isi_hw_status -Lcisfvtbg",
                                "isi_hw_status"),
    'isi_ib_bug_info' : ('exe', "${SUDO}/usr/bin/isi_ib_bug_info", "isi_ib_bug_info"),
    'isi_license'     : ('local_exe', "/usr/bin/isi license", "isi_license"),
    # It's possible that user create lots of exports
    'isi_nfs_exports' : ('local_exe', "/usr/bin/isi nfs exports list --verbose",
                                "isi_nfs_exports"),
    'isi_perfstat'    : ('local_exe', '/usr/bin/isi perfstat', 'isi_perfstat'),
    #MAV 7.0 CLI update
    'isi_smb'         : ('local_exe', "${SUDO}/usr/bin/isi_gather_smb_info", "isi_smb"),
    'isi_stat'        : ('local_exe', "${SUDO}/usr/bin/isi stat -n", "isi_stat"),
    'isi_stat-d'      : ('local_exe', "${SUDO}/usr/bin/isi stat -d -v", "isi_stat-d"),
    'isi_status'      : ('exe', "${SUDO}/usr/bin/isi status", "isi_status"),
    'isi_sync_policy' : ('local_exe', "${SUDO}/usr/bin/isi sync pol list -v",
                                "isi_sync_policy"),
    'isi_ufp'         : ('exe', "/usr/bin/isi_ufp -a", "isi_ufp"),
    'kern.geom'       : ('exe', '/sbin/sysctl kern.geom',
                                'kern.geom'),
    'kern.proc.all_stacks' :
                        ('exe', '/sbin/sysctl kern.proc.all_stacks',
                                'kern.proc.all_stacks'),
    'kern.openfiles'  : ('exe', '/sbin/sysctl kern.openfiles',
                                'kern.openfiles'),
    'kern.mbaudit'    : ('exe', '/sbin/sysctl kern.mbaudit', 'kern.mbaudit'),
    'kern.malloc_pigs': ('exe', "/sbin/sysctl kern.malloc_pigs",
                                "kern.malloc_pigs"),
    'netstat'         : ('exe', "(/usr/bin/netstat -sn; echo; " \
                                "/usr/bin/netstat -rn; echo; " \
                                "/usr/bin/netstat -lan; echo; " \
                                "/usr/bin/netstat -qn; echo; " \
                                "/usr/bin/netstat -m)", "netstat"),
    'node_info_cache' : ('local_tar', "cd /", "node_info_cache.tar " \
                                "/ifs/.ifsvar/db/node_info_cache"),
    'isi_nfs4mgmt'    : ('exe', "/usr/bin/isi_nfs4mgmt", "isi_nfs4mgmt"),
    'nfsstat'         : ('exe', '/usr/bin/isi_gather_nfsstat', 'nfsstat'),
    'override'        : ('tar', "cd /etc/mcp", "override.tar override"),
    'packages'        : ('local_exe', "/usr/bin/isi pkg info", "packages"),
    'packages_local'  : ('exe', "${SUDO}/usr/bin/isi pkg info --forced_local",
                                "packages_local"),
    'rpcinfo'         : ('exe', "/usr/bin/rpcinfo", "rpcinfo"),
    'vfs.nfs'         : ('exe', "/sbin/sysctl vfs.nfs", "vfs.nfs"),
    'vfs.nfsrv'       : ('exe', "/sbin/sysctl vfs.nfsrv", "vfs.nfsrv"),
    'vfs.nlm'         : ('exe', "/sbin/sysctl vfs.nlm", "vfs.nlm"),
    'vm.uma_zalloc_pigs' : ('exe', "/sbin/sysctl vm.uma_zalloc_pigs",
                                   "vm.uma_zalloc_pigs"),
    # Bug 129137 - journal stats for log analysis to see if some code paths
    # can be simplified.
    'journal_stats'   : ('exe', "/sbin/sysctl " \
                                "efs.journal.stats.write.overwrite " \
	                        "efs.journal.stats.write.restarted",
                                "journal_stats"),

    # Don't run quota list on every node.
    #MAV 7.0 - this is being changed to isi_classic until the command:
    #isi quota quota list --format json is suitable to replace it
    'quotas'          : ('local_exe', "/usr/bin/isi_classic quota ls --export",
                                "isi_quota"),
    #MAV 7.0 CLI update
    'quotas_verbose'  : ('local_exe', "/usr/bin/isi quota quotas list  --verbose --format list",
                                "isi_quota_verbose"),
    'sockstat'        : ('exe', "/usr/bin/sockstat", "sockstat"),
    'sin_cache'       : ('exe', "/sbin/sysctl debug.dump_sfm_sin_cache",
                                "sin_cache"),
    'sstore_list'     : ('local_exe', "/usr/sbin/isi_sstore list -l",
                                      "sstore_list"),
    'sstore_stats'    : ('local_exe', "/usr/sbin/isi_sstore stats",
                                      "sstore_stats"),
    'top'             : ('exe', "/usr/bin/top -S -b 1000 -d1", "top"),
    'uname'           : ('exe', "/usr/bin/uname -a", "uname"),
    'uptime'          : ('exe', '/usr/bin/uptime', 'uptime'),
    'varlog_all'      : ('tar', "cd /var", "varlog.tar log"),
    'varlog_recent'   : ('tar', "cd /var",
                         "varlog.tar --exclude '*.gz' --exclude '*.bz2' log"),
    # from /var/crash
    'varcrash_ls'     : ('exe', "${SUDO}ls -lhR /var/crash", "varcrash_ls"),
    'isi_hangdump'    : ('tar', "cd /var/crash",
                         "isi_hangdump.tar isi_hangdump*"),
    'cores'           : ('tar', "cd /var/crash",
                         "cores.tar *.core.gz"),
    # transplanted from isi_hw_bug_info
    'camcontrol'      : ('exe', "${SUDO}/sbin/camcontrol devlist", 'camcontrol'),
    'kern.cam'        : ('exe', "/sbin/sysctl kern.cam", "kern.cam"),
    'hw.mpt0'         : ('exe', "/sbin/sysctl hw.mpt0", "hw.mpt0"),
    'dev.mps'         : ('exe', "/sbin/sysctl dev.mps", "dev.mps"),
    'hw.twe0.report'  : ('exe', "/sbin/sysctl hw.twe0.report",
                                "hw.twe0.report"),
    'isi.idi'         : ('exe', "/sbin/sysctl isi.idi", "isi.idi"),
    'isi.verify'      : ('exe', "/sbin/sysctl isi.verify", "isi.verify"),
    'isi_checkjournal': ('exe', "${SUDO}/usr/sbin/isi_checkjournal -i",
                                "isi_checkjournal"),
    # NOTE potentially unsafe, commented out until resolution of bug 43316.
    #'isi_mptutil'     : ('exe', "isi_mptutil 0 -cmv", 'isi_mptutil'),
    'bmc_sel'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -S -V -p l',
                      'bmc_sel'),
    'bmc_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a bmc',
                      'bmc_info'),
    'car_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a car',
                      'car_info'),
    'cmc_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -d -a cmc',
                      'cmc_info'),
    'car_bootloader_fw' : ('exe',
                           '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -a car -R 2EC0810d',
                           'car_bootloader_fw'),
    'cmc_bootloader_fw' : ('exe',
                           '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -V -a cmc -R 2EC0810d',
                           'cmc_bootloader_fw'),
    'cff_ps1_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -F -V -p D -a cmc -i 1',
                      'cff_ps1_info'),
    'cff_ps2_info' : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_ipmicmc -F -V -p D -a cmc -i 2',
                      'cff_ps2_info'),
    'cdesexputil-chas0-expn0-fw'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -q chas0 expn0',
                      'cdesexputil-chas0-expn0-fw'),
    'cdesexputil-chas0-expn0-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -p all chas0 expn0',
                      'cdesexputil-chas0-expn0-phy'),
    'cdesexputil-chas0-expn0-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_cdesexputil -t all chas0 expn0',
                      'cdesexputil-chas0-expn0-topology'),
    'dsp_config_versions' : ('exe',
                      '/usr/bin/isi_hwtools/isi_dsptool dump_versions',
                      'dsp_config_versions'),
    'isi_mpt_fw_status' : ('exe', '/usr/bin/isi_hwtools/isi_mpt_fw_status '\
                                  '2>/dev/null', 'isi_mpt_fw_status'),
    'isi_mpsutil_fw' : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_mpsutil -q",
                         "isi_mpsutil_fw"),
    'isi_mpsutil_ncq' : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_mpsutil -n",
                         "isi_mpsutil_ncq"),
    'isi_radish'      : ('exe', "${SUDO}/usr/bin/isi_radish -a", "isi_radish"),
    'isi_radish_bootdisk' : ('exe',
                         "${SUDO}/usr/bin/isi_radish -a 'Internal J3' 'Internal J4' 'Carrier board J3' 'Carrier board J4'",
                         "isi_radish_bootdisk"),
    'isi_smbios'      : ('exe', "${SUDO}/usr/bin/isi_smbios", "isi_smbios"),
    'isi_twe_config'  : ('exe', "${SUDO}/usr/bin/isi_tweconfig -CC", "isi_twe_config"),
    'lsiexputil-chas0-expn0-fw'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -q chas0 expn0',
                      'lsiexputil-chas0-expn0-fw'),
    'lsiexputil-chas0-expn0-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -p all chas0 expn0',
                      'lsiexputil-chas0-expn0-phy'),
    'lsiexputil-chas0-expn0-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -t all chas0 expn0',
                      'lsiexputil-chas0-expn0-topology'),
    'lsiexputil-chas0-expn0-log'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -l chas0 expn0',
                      'lsiexputil-chas0-expn0-log'),
    'lsiexputil-chas0-expn1-fw'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -q chas0 expn1',
                      'lsiexputil-chas0-expn1-fw'),
    'lsiexputil-chas0-expn1-phy'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -p all chas0 expn1',
                      'lsiexputil-chas0-expn1-phy'),
    'lsiexputil-chas0-expn1-topology'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -t all chas0 expn1',
                      'lsiexputil-chas0-expn1-topology'),
    'lsiexputil-chas0-expn1-log'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_lsiexputil -l chas0 expn1',
                      'lsiexputil-chas0-expn1-log'),
    'isi_fcb_vpd_dump'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_fcb_vpd_tool dump',
                      'isi_fcb_vpd_dump'),
    'isi_fcb_vpd_dumpall'  : ('exe',
                      '${SUDO}/usr/bin/isi_hwtools/isi_fcb_vpd_tool --devid all dump',
                      'isi_fcb_vpd_dumpall'),
    'isi_fcb_vpd_fdump'  : ('exe', '${SUDO}/usr/bin/isi_hwtools/isi_fcb_vpd_tool fdump',
                            'isi_fcb_vpd_fdump'),
    'gmirror_list'    : ('exe', '/sbin/gmirror list',
                               'gmirror_list'),
    'gmirror_status'  : ('exe', '/sbin/gmirror status',
                               'gmirror_status'),
    'gpart_show'  : ('exe', '/sbin/gpart show', 'gpart_show'),
    'isi_lox_info'  : ('exe', '/usr/bin/isi_lox_info -d', 'isi_lox_info'),
    'isi_bootdisk_status'  : ('exe', '${SUDO}/usr/bin/isi_bootdisk_status',
                               'isi_bootdisk_status'),
    'pciconf'      : ('exe', "${SUDO}/usr/sbin/pciconf -lbcv", "pciconf"),
    'vitutil-chas0-fw'  : ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -q chas0',
                            'vitutil-chas0-fw'),
    'vitutil-chas0-conf': ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -cvm chas0',
                            'vitutil-chas0-conf'),
    'vitutil-chas1-conf': ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -cvm chas1',
                            'vitutil-chas1-conf'),
    'vitutil-chas1-fw'  : ('exe',
                            '${SUDO}/usr/bin/isi_gather_vitutil_info -q chas1',
                            'vitutil-chas1-fw'),
    'isi_fputil'      : ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_fputil -q",
                         "isi_fputil"),
    'dev.isp'         : ('exe', "/sbin/sysctl dev.isp", "dev.isp"),
    'isi_fc'          : ('exe', "${SUDO}/usr/bin/isi fc list", "isi_fc"),
    'isi_tape'        : ('exe', "${SUDO}/usr/bin/isi tape list -v", "isi_tape"),
    'efs.bam.dump_last_write_plan':
                        ('exe', "/sbin/sysctl efs.bam.dump_last_write_plan",
                                    'efs.bam.dump_last_write_plan'),
    'minidumps'       : ('tar', "cd /var/crash", "minidumps.tar vmcore*.*"),
    'lastgather_timestamp': ('local_exe', \
                             'ls -lT /ifs/data/Isilon_Support/*gather '\
                             ' /var/crash/support/*gather ' , \
                             'lastgather_timestamp'),
    'isi_statistics_pstat':
        ('exe',       "/usr/bin/isi statistics pstat", "isi_statistics_pstat"),

    # these have been added primarily for the SupportIQ agent.
    # all nodes
    'df_all':
        ('exe',       "/bin/df -hi", "df_all"),
    'isi_auth_nodes':
        ('exe', "${SUDO}/usr/bin/isi_gather_auth_info -n", "isi_auth"),
    'isi_client_stats':
        ('exe',       "/usr/bin/isi_client_stats --notop -r 10",
            "isi_client_stats"),
    'iostat':
        ('exe',       "/usr/sbin/iostat -d -n 64", "iostat"),
    'isi_flexprotect':
        ('exe',       "/usr/bin/isi flexprotect", "isi_flexprotect"),
    'isi_hw_status_all':
        ('exe',      "${SUDO}/usr/bin/isi_hw_status -A", "isi_hw_status_all"),
    'isi_services':
        ('exe',      "${SUDO}/usr/bin/isi services -a", "isi_services"),
    'isi_testjournal':
        ('exe', "/usr/sbin/isi_testjournal", "isi_testjournal"),
    'netstat_listen_queue':
        ('exe',       "/usr/bin/netstat -nL", "netstat_listen_queue"),
    'netstat_routes':
        ('exe',       "/usr/bin/netstat -anr", "netstat_routes"),
    'netstat_sockets':
        ('exe',       "/usr/bin/netstat -an", "netstat_sockets"),
    'ps':
        ('exe',       "/bin/ps auxww", "ps"),
    'ps_full':
        ('exe',       "/bin/ps auxwwHl", "ps_full"),
    'resolv.conf':
        ('exe',       "/bin/cat /etc/resolv.conf", "resolv.conf"),

    # single node only
    'ib_subnet_master_local':
        ('local_exe', "/usr/bin/grep ': MA :' /var/log/opensm*.log",
                                "ib_subnet_master"),
    'isi_alerts_local':
        ('local_exe', "/usr/bin/isi events list -w", "isi_alerts"),
    'isi_alerts_history':
        ('local_exe', "/usr/bin/isi events list -w -s", "isi_alerts_history"),
    'isi_auth':
        ('local_exe', "${SUDO}/usr/bin/isi_gather_auth_info", "isi_auth"),
    # no --timeout as this is really an isi_classic command
    'isi_auth_mapping_dump':
        ('local_exe', "${SUDO}/usr/bin/isi auth mapping dump",
            "isi_auth_mapping_dump"),
    'isi_auth_roles_list':
        ('local_exe',
            "${SUDO}/usr/bin/isi --timeout=15 auth roles list --verbose",
            "isi_auth_roles_list"),
    'isi_flexprotect_local':
        ('local_exe', "/usr/bin/isi flexprotect", "isi_flexprotect"),
    'isi_ib_bug_info_local':
        ('local_exe', "/usr/bin/isi_ib_bug_info", "isi_ib_bug_info"),
    'isi_nfs_exports_local' :
        ('local_exe', "/usr/bin/isi nfs exports list --verbose",
            "isi_nfs_exports"),
    'isi_statistics':
        ('local_exe', "/usr/bin/isi statistics system -d --nodes",
             "isi_statistics"),
    'isi_statistics_lsass':
        ('local_exe', "/usr/bin/isi statistics protocol --degraded --timeout 5 --long --protocols=lsass_in,lsass_out",
             "isi_statistics_lsass"),
    'isi_statistics_nfs':
        ('local_exe', "/usr/bin/isi statistics protocol --degraded --timeout 5 --long --protocols=nfs3,nfs4",
             "isi_statistics_nfs"),
    'isi_statistics_smb':
        ('local_exe', "/usr/bin/isi statistics protocol --degraded --timeout 5 --long --protocols=smb1,smb2",
             "isi_statistics_smb"),
    'isi_sync_policy_local':
        ('local_exe', "/usr/bin/isi sync pol ls -v", "isi_sync_policy"),
    'isi_sync_job_local':
        ('local_exe', "/usr/bin/isi sync job ls", "isi_sync_job"),
    'isi_radish_local':
        ('local_exe', "/usr/bin/isi_radish -a", "isi_radish"),
    'isi_job_status':
        ('local_exe', "/usr/bin/isi_classic job status -v", "isi_job_status"),
    'isi_job_history':
        ('local_exe', "/usr/bin/isi_classic job history -v -l 0", "isi_job_history"),
    'isi_job_list':
        ('local_exe', "/usr/bin/isi_classic job list", "isi_job_list"),
    'isi_job_schedule':
        ('local_exe', "/usr/bin/isi_classic job schedule list", "isi_job_schedule"),
    'isi_job_policy':
        ('local_exe', "${SUDO}/usr/bin/isi_classic job policy list", "isi_job_policy"),
    'isi_job_config':
        ('local_exe', "${SUDO}/usr/bin/isi_classic job config", "isi_job_config"),
    'isi_networks_local':
        ('local_exe', "/usr/bin/isi networks", "isi_networks"),
    'isi_networks_interfaces_local':
        ('local_exe', "/usr/bin/isi networks ls ifaces -v",
                "isi_networks_interfaces"),
    'isi_networks_rules_local':
        ('local_exe', "/usr/bin/isi networks ls rules -v", "isi_networks_rules"),
    'isi_networks_pools_local':
        ('local_exe', "/usr/bin/isi networks ls pools -v",
                "isi_networks_pools"),
    'isi_networks_subnets_local':
        ('local_exe', "/usr/bin/isi networks ls subnets -v",
                "isi_networks_subnets"),
    'isi_services_local':
        ('local_exe', "/usr/bin/isi services -a", "isi_services"),
    'isi_status_local':
        ('local_exe', "/usr/bin/isi status", "isi_status"),
    'isi_tape_local':
        ('local_exe', "/usr/bin/isi tape list -v", "isi_tape"),
    # Grab pstore, but remove their current AD password for customer security
    'pstore_dump':
        ('local_exe', "grep -v 'Password.value' /ifs/.ifsvar/pstore.gc",
            "pstore_dump"),
    'pstore_changes_dump':
        ('local_exe',
            "grep -v 'Password.value' /ifs/.ifsvar/pstore_changes.log",
            "pstore_changes_dump"),
    'snapshot_list_local':
        ('local_exe', "/usr/bin/isi snap list", "snapshot_list"),
    'snapshot_schedule_local':
        ('local_exe', "/usr/bin/isi snap schedule list", "snapshot_schedule"),
    'snapshot_settings_local':
        ('local_exe', "/usr/bin/isi snap settings view", "snapshot_settings"),
    'snapshot_usage_local':
        ('local_exe', "/usr/bin/isi snap snap list -v --format table --state all",
            "snapshot_usage"),
    'uptime_local':
        ('local_exe', '/usr/bin/uptime', 'uptime'),
    'varlog_messages':
        ('tar', "cd /var/log", "messages.tar messages"),
    'etc_newsyslog':
        ('tar', "cd /etc", "newsyslog_conf.tar newsyslog*"),
    #added for MAV 7.0
    'efs.bam.ec.stats' :
        ('exe', "/sbin/sysctl efs.bam.ec.stats", 'efs.bam.ec.stats'),
    #added for MAV 7.0
    'efs.bam.ec.known_logstores' :
            ('exe', "/sbin/sysctl efs.bam.ec.known_logstores", 'efs.bam.ec.known_logstores'),
    'audit_settings':
        ('local_exe', "/usr/bin/isi audit settings view", "isi_audit_settings"),
    'audit_topics':
        ('local_exe', "/usr/bin/isi audit topics list", "isi_audit_topics"),
    'smartlock_domain_long':
        ('local_exe', "/usr/bin/isi worm domains list -v", "isi_worm_domains_long"),
    'smartlock_domain_wide':
        ('local_exe', "/usr/bin/isi worm domains list", "isi_worm_domains_wide"),
    'efs.lbm' :
        ('exe', "/sbin/sysctl efs.lbm", 'efs.lbm'),
    'efs.bam.bsw' :
        ('exe', "/sbin/sysctl efs.bam.bsw", 'efs.bam.bsw'),
    'dmesg_boot':
        ('tar', "cd /var/run", "dmesg.tar dmesg.boot*"),
    'efs.gmp.current_info': ('exe', '/sbin/sysctl efs.gmp.current_info', 'efs.gmp.current_info'),
    'isi_filepool_policies':
        ('local_exe', "/usr/bin/isi filepool policies list -v", "isi_filepool_policies"),
    'efs.bam.layout.ssd.sys_btree.mirrors' :
        ('exe', "/sbin/sysctl efs.bam.layout.ssd.sys_btree.mirrors", 'efs.bam.layout.ssd.sys_btree.mirrors'),
    'isi_filepool_default_policies':
        ('local_exe', "/usr/bin/isi filepool default-policy view", "isi_filepool_default_policies"),
    'isi_hw_check':
        ('exe', "${SUDO}/usr/bin/isi_hwtools/isi_hw_check -s -m safe.id.dram", "isi_hw_check"),

}

# Command line option can replace 'varlog_all' with 'varlog_recent'
EXEMPT = [
    'varlog_all',    # always must be first

    'abr_config',
    'aspera_config',
    'atacontrol',
    'camcontrol',
    'celog_node_db',
    'celog_ifsvar_db',
    'configs',
    'cto_summary',
    'dedupe_reports',
    'dedupe_settings',
    'dedupe_stats',
    'dev.isp',
    'compliance',
    'df',
    'du',
    'efs.bam.dump_last_write_plan',
    'efs.bam.ec.stats',
    'efs.bam.bsw',
    'efs.bam.ec.known_logstores',
    'efs.gmp.group',
    'efs.l3',
    'efs.lbm.drive_space',
    'efs.lbm',
    'efs.snapshot',
    'efs.stf',
    'etcifs',
    'etc',
    'fstat',
    'gconfig_local',
    'gconfig_ifsvar',
    'hw.ata',
    'hw.disk_event',
    'hw.ecclist',
    'hw.iosched',
    'hw.mpt0',
    'hw.twe0.report',
    'dev.mps',
    'ifconfig',
    'ifsvar_db_pkg',
    'ifsvar_db_pkg_tars',
    'ifsvar_etc',
    'ifsvar_iscsi',
    'ifsvar_modules',
    'ifsvar_tmp',
    'ifsvar_modules_jobengine_cp',
    'isi.idi',
    'isi.verify',
    'isi_alerts',
    'isi_alerts_history',
    'isi_auth',
    'isi_auth_nodes',
    'isi_auth_mapping_dump',
    'isi_auth_roles_list',
    'isi_checkjournal',
    'isi_devices',
    'isi_devices_fwstatus',
    'isi_drivefirmware',
    'isi_drivenum',
    'isi_disk_inject',
    'isi_diskpool',
    'efs.bam.disk_pool_db',
    'isi_dmi_log',
    'isi_fc',
    'isi_hangdump',
    'cores',
    'isi_hw_status',
    'isi_ib_bug_info',
    'isi_iscsi_target',
    'isi_job_status',
    'isi_job_history',
    'isi_job_list',
    'isi_job_schedule',
    'isi_job_policy',
    'isi_job_config',
    'isi_license',
    'isi_networks_local',
    'isi_networks_interfaces_local',
    'isi_networks_rules_local',
    'isi_networks_pools_local',
    'isi_networks_subnets_local',
    'isi_nfs_exports',
    'isi_perfstat',
    'isi_radish',
    'isi_radish_bootdisk',
    'isi_services_local',
    'isi_smb',
    'isi_smbios',
    'isi_stat',
    'isi_statistics',
    'isi_statistics_drive',
    'isi_statistics_lsass',
    'isi_statistics_nfs',
    'isi_statistics_smb',
    'isi_status',
    'isi_stat-d',
    'isi_sync_policy',
    'isi_tape',
    'isi_ufp',
    'isi_twe_config',
    'kern.cam',
    'kern.geom',
    'kern.proc.all_stacks',
    'kern.malloc_pigs',
    'kern.mbaudit',
    'kern.openfiles',
    'lastgather_timestamp',
    'minidumps',
    'netstat',
    'isi_nfs4mgmt',
    'nfsstat',
    'node_info_cache',
    'override',
    'packages',
    'packages_local',
    'ps',
    'ps_full',
    'pstore_changes_dump',
    'pstore_dump',
    'rpcinfo',
    'resolv.conf',
    'quotas',
    'quotas_verbose',
    'sockstat',
    'snapshot_list_local',
    'snapshot_schedule_local',
    'snapshot_settings_local',
    'snapshot_usage_local',
    'sin_cache',
    'sstore_list',
    'sstore_stats',
    'top',
    'uname',
    'uptime',
    'varcrash_ls',
    'isi_mpt_fw_status',
    'bmc_sel',
    'bmc_info',
    'car_info',
    'cmc_info',
    'car_bootloader_fw',
    'cmc_bootloader_fw',
    'cff_ps1_info',
    'cff_ps2_info',
    'cdesexputil-chas0-expn0-fw',
    'cdesexputil-chas0-expn0-phy',
    'cdesexputil-chas0-expn0-topology',
    'dsp_config_versions',
    'isi_mpsutil_fw',
    'isi_mpsutil_ncq',
    'lsiexputil-chas0-expn0-fw',
    'lsiexputil-chas0-expn0-phy',
    'lsiexputil-chas0-expn0-log',
    'lsiexputil-chas0-expn0-topology',
    'lsiexputil-chas0-expn1-fw',
    'lsiexputil-chas0-expn1-phy',
    'lsiexputil-chas0-expn1-log',
    'lsiexputil-chas0-expn1-topology',
    'isi_fcb_vpd_dump',
    'isi_fcb_vpd_dumpall',
    'isi_fcb_vpd_fdump',
    'gmirror_list',
    'gmirror_status',
    'gpart_show',
    'isi_bootdisk_status',
    'pciconf',
    'vfs.nfs',
    'vfs.nfsrv',
    'vfs.nlm',
    'vitutil-chas0-conf',
    'vitutil-chas0-fw',
    'vitutil-chas1-conf',
    'vitutil-chas1-fw',
    'isi_fputil',
    'vm.uma_zalloc_pigs',
    'journal_stats',
    'etc_newsyslog',
    'isi_lox_info',
    'audit_settings',
    'audit_topics',
    'smartlock_domain_long',
    'smartlock_domain_wide',
    'isi_statistics_pstat',
    'zsh_history',
    'efs.gmp.current_info',
    'isi_filepool_policies',
    'isi_filepool_default_policies',
    'efs.bam.layout.ssd.sys_btree.mirrors',
    'isi_hw_check',
    'zsh_history_compadmin',
]

# GROUPS holds the groups of utilities that can be specified by the
# --group command line option.
GROUPS = {
    "abr": [
        "abr_config",
        "cto_summary",
    ],
    "cluster": [
        "isi_status",
        "isi_alerts_local",
        "isi_services_local",
        "isi_status_local",
        "isi_license",
        "isi_tape_local",
        "packages",
        "uname",
        #"isi_firmware",     # unsafe, unimplemented; bug 43316
    ],
    "fs": [
        "isi_checkjournal",
        "isi_testjournal",
        "efs.gmp.group",
        "isi_job_status",
        "isi_job_history",
        "isi_job_list",
        "isi_job_schedule",
        "isi_job_policy",
        "isi_flexprotect_local",
    ],
    "auth": [
        "isi_auth",
        "isi_auth_nodes",
        "isi_auth_mapping_dump",
        "isi_auth_roles_list",
        "pstore_changes_dump",
        "pstore_dump",
        "isi_statistics_lsass",
    ],
    "protocol": [
        "isi_nfs_exports_local",
        "isi_smb",
        "isi_statistics_nfs",
        "isi_statistics_smb",
        "isi_nfs4mgmt",
        "nfsstat",
        "ps_full",
        "rpcinfo",
        "vfs.nfs",
        "vfs.nfsrv",
        "vfs.nlm",
    ],
    "messages": [
        "varlog_messages",
    ],
    "bootmessages": [
        "dmesg_boot",
    ],
    "network": [
        "ifconfig",
        "resolv.conf",
        "netstat_listen_queue",
        "netstat_sockets",
        "netstat_routes",
        "sockstat",
        "isi_client_stats",
        "isi_networks_local",
        "isi_networks_interfaces_local",
        "isi_networks_rules_local",
        "isi_networks_pools_local",
        "isi_networks_subnets_local",
    ],
    "application": [
        "snapshot_list_local",
        "snapshot_usage_local",
        "snapshot_schedule_local",
        "snapshot_settings_local",
        "efs.snapshot",
        "efs.stf",
        "quotas_verbose",	# quota is local only command now
        "isi_sync_policy_local",
        "isi_sync_job_local",
    ],
    "ib": [
        "isi_ib_bug_info_local",
        "ib_subnet_master_local",
    ],
    "node": [
        'isi_status',
        'bmc_info',
        'car_info',
        'cmc_info',
        'car_bootloader_fw',
        'cmc_bootloader_fw',
        'cff_ps1_info',
        'cff_ps2_info',
        'dsp_config_versions',
        "isi_devices",
        "isi_devices_fwstatus",
        "isi_drivenum",
        'isi_fcb_vpd_dump',
        'isi_fcb_vpd_dumpall',
        'isi_fcb_vpd_fdump',
        "isi_hw_status_all",
        "atacontrol",
        "camcontrol",
        "isi_dmi_log",
        "uptime_local",
        "isi_radish_local",
        "isi_radish_bootdisk",
        "isi_bootdisk_status",
        "gmirror_list",
        "gmirror_status",
        "gpart_show",
        "hw.ata",
        "df_all",
        "varcrash_ls",
        "kern.geom",
        "kern.proc.all_stacks",
        "pciconf",
        "etc_newsyslog",
        "isi_lox_info",
        "cores",
    ],
    "storage": [
        "atacontrol",
        "camcontrol",
        'cdesexputil-chas0-expn0-fw',
        'cdesexputil-chas0-expn0-phy',
        'cdesexputil-chas0-expn0-topology',
        "hw.ata",
        "hw.mpt0",
        "dev.mps",
        "hw.disk_event",
        "hw.ecclist",
        "hw.iosched",
        "kern.cam",
        "gmirror_list",
        "gmirror_status",
        "gpart_show",
        "isi_bootdisk_status",
        "isi_radish_bootdisk",
        "isi_devices",
        "isi_devices_fwstatus",
        "isi_drivefirmware",
        "isi_drivenum",
        "isi_radish",
        "isi_mpt_fw_status",
        "isi_mpsutil_fw",
        "isi_mpsutil_ncq",
        "lsiexputil-chas0-expn0-fw",
        "lsiexputil-chas0-expn0-phy",
        "lsiexputil-chas0-expn0-topology",
        "lsiexputil-chas0-expn0-log",
        "lsiexputil-chas0-expn1-fw",
        "lsiexputil-chas0-expn1-phy",
        "lsiexputil-chas0-expn1-topology",
        "lsiexputil-chas0-expn1-log",
        "vitutil-chas0-fw",
        "vitutil-chas0-conf",
        "vitutil-chas1-conf",
        "vitutil-chas1-fw",
    ],
    "usage": [
        "uptime",
        "top",
        "isi_statistics",
        "fstat",
        "kern.openfiles",
        "kern.mbaudit",
        "iostat",
    ],
    "logs": [
        "varlog_recent",
    ],
}

# Commands for email body. node means executed on local node,
# cluster means cluster wide.
MAIL_COMMAND = [
        ('uname -a', 'node'),
        ('isi_hw_status -i', 'cluster'),
        ('isi status', 'node'),
        ('isi status -n', 'node'),
        ('isi event list -w', 'node'),
        ('isi_hw_status -A', 'cluster'),
        ('isi_bootdisk_status', 'cluster'),
        ('isi devices', 'cluster'),
        ('date; uptime', 'cluster'),
        ('df', 'cluster'),
        ('isi_classic job status -v', 'node'),
        ('ps auxcjl', 'cluster'),
        ('ifconfig -a', 'cluster')
]


class Cluster: pass
class Options: pass

# Globals
cluster = Cluster()
options = Options()
log = ''    #used later on to write to gather-status log file
re_upload = ''

def is_ifs_writeable():
    global SUPPORT_DIR

    if not os.path.exists(SUPPORT_DIR):
        parentdir = os.path.dirname(SUPPORT_DIR.rstrip("/"))
        try:
            os.utime(parentdir, (-1, -1))
        except OSError, e:
            if e.errno == errno.EROFS:
                print "Read-only file system: %s. Attempting /var/crash." % SUPPORT_DIR
            else:
                # some error, still bail out
                print "Attempting /var/crash as %s has OS error: %s" % \
                    (parentdir, os.strerror(e.errno))

            return False
    else:
        try:
            os.utime(SUPPORT_DIR, (-1, -1))
        except OSError, e:
            if e.errno == errno.EROFS:
                print "Read-only file system: %s. Attempting /var/crash." % SUPPORT_DIR
            else:
                print "Attempting /var/crash as %s has OS error: %s" % \
                    (parentdir, os.strerror(e.errno))

            return False

    return True

def update_support_dirs(options):
    global SUPPORT_DIR
    global TMP_SUPPORT_DIR
    global PKG_SUPPORT_DIR
    global GATHER_SCRIPT
    global LOCAL_GATHER_SCRIPT
    global ISI_NODES_OUTPUT
    global ISI_FIRMWARE_OUTPUT
    global CELOG_EVENTS_INFO
    global PACKAGE_INFO
    global SAVED_COMMAND_LINE
    global LOCAL_TMP_SUPPORT_DIR
    global INCREMENTAL_FILE
    global FULLGATHER_FILE
    global GATHER_RUN_FILE
    global IFS_READONLY

    if options.tempdir:
        if options.tempdir[0] != '/':
            error("Only full path tempdir specifications allowed.")
        SUPPORT_DIR = options.tempdir
    elif options.use_ifs:
        SUPPORT_DIR = IFS_SUPPORT_DIR
        if not is_ifs_writeable():
            IFS_READONLY = True
            SUPPORT_DIR = VAR_SUPPORT_DIR
            options.use_ifs = False
    else:
        SUPPORT_DIR = VAR_SUPPORT_DIR

    # generate a unique random tempdir. this is not safe against races, but
    # shouldn't need to be.
    chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
    while True:
        tmpdir = ""
        for i in range(16):
            tmpdir += chars[ random.randint(0, len(chars)-1) ]
        # this directory is expected to end in "/" elsewhere in the code, but
        # to verify that a file doesn't exist by the name of the directory
        # we wish to create, we have to check if the directory exists without
        # the trailing slash, first.
        TMP_SUPPORT_DIR = os.path.join(SUPPORT_DIR, _TMP_SUPPORT_DIR, tmpdir)
        if not os.path.exists(TMP_SUPPORT_DIR):
            TMP_SUPPORT_DIR  += "/"
            break

    if options.tardir:
        PKG_SUPPORT_DIR   = options.tardir
    else:
        PKG_SUPPORT_DIR   = os.path.join(SUPPORT_DIR, _PKG_SUPPORT_DIR)


    INCREMENTAL_FILE      = os.path.join(SUPPORT_DIR, _INCREMENTAL_FILE)
    FULLGATHER_FILE       = os.path.join(SUPPORT_DIR, _FULLGATHER_FILE)
    GATHER_SCRIPT         = os.path.join(TMP_SUPPORT_DIR, _GATHER_SCRIPT)
    LOCAL_GATHER_SCRIPT   = os.path.join(TMP_SUPPORT_DIR, _LOCAL_GATHER_SCRIPT)
    ISI_NODES_OUTPUT      = os.path.join(TMP_SUPPORT_DIR, _ISI_NODES_OUTPUT)
    ISI_FIRMWARE_OUTPUT   = os.path.join(TMP_SUPPORT_DIR, _ISI_FIRMWARE_OUTPUT)
    CELOG_EVENTS_INFO     = os.path.join(TMP_SUPPORT_DIR, _CELOG_EVENTS_INFO)
    PACKAGE_INFO          = os.path.join(TMP_SUPPORT_DIR, _PACKAGE_INFO)
    SAVED_COMMAND_LINE    = os.path.join(TMP_SUPPORT_DIR, _SAVED_COMMAND_LINE)
    LOCAL_TMP_SUPPORT_DIR = os.path.join(TMP_SUPPORT_DIR, _LOCAL_TMP_SUPPORT_DIR)
    GATHER_RUN_FILE_NAME = 'gather-%d-%d.xml' % \
            (array.getLocalId(), os.getpid())
    GATHER_RUN_FILE = os.path.join(RUN_SUPPORT_DIR, GATHER_RUN_FILE_NAME)

    # Make sure the dirs exist locally first
    makeDir(SUPPORT_DIR)
    makeDir(TMP_SUPPORT_DIR)
    makeDir(LOCAL_TMP_SUPPORT_DIR)
    makeDir(PKG_SUPPORT_DIR)

    if options.symlinkdir:
        makeDir(options.symlinkdir)

def writeGatherRunFile():
    global TMP_SUPPORT_DIR
    global GATHER_RUN_FILE

    try:
        grfd = os.open(GATHER_RUN_FILE,
                os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0755)

    except (OSError, IOError), e:
        verbose('Unable to create run file on ifs. Attempting var.')
        makeDir(RUN_VAR_SUPPORT_DIR)
        GATHER_RUN_FILE = GATHER_RUN_FILE.replace(RUN_SUPPORT_DIR,
                    RUN_VAR_SUPPORT_DIR) 
        try:
            grfd = os.open(GATHER_RUN_FILE,
                    os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0755)
        except (OSError, IOError), e:
            error('Could not create gather run file:' + str(e))

    grfile = os.fdopen(grfd, 'w')

    root = etree.Element('gather-run')
    tmpdir = etree.SubElement(root, 'tmpdir')
    tmpdir.text = TMP_SUPPORT_DIR
    et = etree.ElementTree(root)
    et.write(grfile, arrcfg.config_load().encoding)

def cleanupGatherRunFile():
    try:
        os.unlink(GATHER_RUN_FILE)
    except Exception, e:
        verbose('Could not cleanup gather run file: ' + str(e))

# this routine creates both the _gather script that is run by all nodes and
# the local_gather script that is only called once per cluster, on the
# invoking node.  run_utils should be a list of the names of utilities in
# the UTILITIES dict.
def generate_scripts(run_utils):
    # Determine if cluster is in compliance mode
    try:
        flags    = os.O_RDWR|os.O_CREAT|os.O_TRUNC
        header = "#!/usr/local/bin/bash\nHOST=`hostname`\n"
        if is_compliance():
            header += "SUDO='sudo '\n"

        localfd  = os.open(LOCAL_GATHER_SCRIPT, flags, 0755)
        os.write(localfd, header)

        if not options.local_only:
            gatherfd = os.open(GATHER_SCRIPT, flags, 0755)
            os.write(gatherfd, header)

        for util in run_utils:
            exe,cmd,name = UTILITIES.get(util)
            if exe == 'exe' and not options.local_only:
                line = "%s 1> %s$HOST/%s 2>&1" % \
                    (cmd, TMP_SUPPORT_DIR, name.lstrip())
            elif exe == 'tar' and not options.local_only:
                if options.incremental:
                    line = "%s;/usr/bin/tar -c --newer-mtime-than %s -f %s$HOST/%s 1>/dev/null 2>&1" % \
                        (cmd, INCREMENTAL_FILE, TMP_SUPPORT_DIR, name.lstrip())
                    # Remove the tar file if it's 0 bytes in length
                    TarfileName=name.split(' ')[0]
                    TarfileName= "%s$HOST/%s" % \
                        (TMP_SUPPORT_DIR, TarfileName)
                    line += "\n"
                    line += '[ -s "%s" ] || rm "%s"' % \
                        (TarfileName, TarfileName)
                else:
                    line = "%s;/usr/bin/tar cf %s$HOST/%s 1>/dev/null 2>&1" % \
                        (cmd, TMP_SUPPORT_DIR, name.lstrip())
            elif exe == 'local_tar' or (options.local_only and exe == 'tar'):
                if options.incremental:
                    line = "%s;/usr/bin/tar -c --newer-mtime-than %s -f %slocal/%s 1>/dev/null 2>&1" % \
                        (cmd, INCREMENTAL_FILE, TMP_SUPPORT_DIR, name.lstrip())
                    # Remove the tar file if it's 0 bytes in length
                    TarfileName=name.split(' ')[0]
                    TarfileName= "%slocal/%s" % (TMP_SUPPORT_DIR, TarfileName)
                    line += "\n"
                    line += '[ -s "%s" ] || rm "%s"' % \
                        (TarfileName, TarfileName)
                else:
                    line = "%s;/usr/bin/tar cf %slocal/%s 1>/dev/null 2>&1" % \
                        (cmd, TMP_SUPPORT_DIR, name.lstrip())
            elif exe == 'local_exe' or (options.local_only and exe == 'exe'):
                line = "%s 1> %slocal/%s 2>&1" % \
                    (cmd, TMP_SUPPORT_DIR, name.lstrip())
            else:
                line = ''

            line += "\n"

            # if the util type begins "local_" it goes to the local script.
            if exe.find("local_") == 0 or options.local_only:
                os.write(localfd, line)
            else:
                os.write(gatherfd, line)

        # script must exit cleanly
        footer = "exit 0\n"

        os.write(localfd, footer)
        os.close(localfd)

        if not options.local_only:
            os.write(gatherfd, footer)
            os.close(gatherfd)

    except OSError, (errno, strerror):
        error("OS error: %s" % os.strerror(errno))

def is_compliance():
    try:
        res = (sysctlbyname('isi.compliance') == 1)
    except Exception, e:
        res = False
    return res

#
# the core of this utility is basically isi_rdo, since that did
# the majority of what i wanted to do.
#
# assumptions:
#   - the script will be run on the quorum-side of the cluster (such
#   that ifs is writeable).
#   - the cluster has enough free space to store the temporary info
#   files and the final compressed package (about 500MB needed, usually
#   determined by the size of /var/log and the bug_infos).
#   - the cluster has enough wherewithal to actually gather the required
#   logs.
#

def main():
    global cluster, options, log, re_upload, IFS_READONLY

    activityIndicator = Dots()
    log = Logger(GATHER_STATUS_PATH)
    #
    # don't treat ^C special
    # and ignore the HUP signal
    signal.signal(signal.SIGINT, signal.SIG_DFL)
    signal.signal(signal.SIGHUP, signal.SIG_IGN)

    # internal correctness check
    validateUtils()

    cluster.name     = None
    cluster.hostname = None
    cluster.user     = None
    cluster.password = None
    cluster.nodes    = []

    query           = False

    incremental_sequence = 0    # number of incremental gathers since full

    options.extra_file_list   = []
    options.gather_script     = []
    options.gather_expr       = []
    options.gather_expr_local = []
    options.analysis_script   = []
    options.analysis_expr     = []

    options.isi_nodes_args  = []
    options.includes        = []
    options.incremental     = False
    options.local_only      = False
    options.skip_node_check = False
    options.symlinkdir      = None
    options.tardir          = None
    options.tarfile         = None
    options.tempdir         = None
    options.use_ifs         = True
    options.no_exempt_logs  = False
    options.debug           = False
    options.verbose         = False
    options.upload_package  = True

    options.group           = []
    options.group_utils     = []
    options.ftp_enable      = True
    options.ftp_host        = 'ftp.isilon.com'
    options.ftp_user        = 'anonymous'
    options.ftp_pass        = ' '
    options.ftp_path        = "/incoming"
    options.ftp_port        = 21
    options.ftp_proxy_host  = ''
    options.ftp_proxy_port  = ''
    options.try_active_ftp  = True
    options.try_passive_ftp = True

    options.email_enable    = False
    options.email_addresses = ''
    options.email_from      = ''
    options.email_subject   = ''
    options.email_body      = ''
    options.skip_size_check  = False

    options.http_enable     = True
    options.http_host        = ''
    options.http_path        = ''
    options.http_proxy_host = ''
    options.http_proxy_port = ''

    options.esrs            = False
    options.clean_cores     = False
    options.clean_dumps     = False
    options.dumps           = False
    options.cores           = False
    options.save_only       = False
    options.verify_upload   = False
    options.re_upload       = False

    #
    # Allow all directory and data creation to be accessed by the group users
    #
    os.umask(0002)

    #
    # parse options
    #
    get_options()

    if options.save_only:
        saveConfigOptions()
        sys.exit(0)

    if options.verify_upload:
        verifyConnection()
        sys.exit(0)

    if options.re_upload:
        reUploadPackage(re_upload)
        sys.exit(0)

    if options.use_ifs and cluster.password:
        print "\nNote: A password is only required with -L or -T.  Ignoring."

    # update paths
    update_support_dirs(options)
    writeGatherRunFile()

    if len(options.analysis_expr):
        error("post-gather analysis expressions not currently supported")

    #
    # Currently we do not support an incremental gather file on local
    # filesystems, only on /ifs.
    #
    if options.incremental and not options.use_ifs:
        error("Incremental gather supported only on /ifs, not local filesystem")


    # do not support --group with --incremental, as incremental only reall
    # makes sense in relation to a full log set.
    if options.incremental and options.group:
        error("Cannot specify both --incremental and --group")

    #
    # query for the password if files are not saved to IFS.
    # Note we query BEFORE attempting any remote node activity,
    # such as makeDirs() call below.
    #
    if cluster.password == None and not options.use_ifs \
           and not options.local_only:
        if IFS_READONLY:
            cluster.password = getpass.getpass(prompt="Root password is required when /ifs is " \
                                   "a Read-only file system.\nPassword:")
        else:
            cluster.password = getpass.getpass()

    # first, verify that the node we're being run from is part of the
    # quorum-side of the cluster.
    #
    if options.use_ifs and not hasQuorum():
        error("This node does not have quorum -- OneFS is not writeable.  Aborting.")

    # Should we verify SUPPORT_DIR accessibility here?

    cluster.name, cluster.hostname = getHostname()

    #
    # figure out which nodes to run on.  first, though, determine which
    # nodes may be down.  we do this so that the customer need not
    # supply the -x semantic on the command line, since this may be
    # difficult for the customer to know or understand, especially in
    # larger clusters.
    #
    # since the system reports down devices based on device ID, but the
    # isi_rdo logic for -x uses LNN, we need to map b/w ID and
    # LNN.
    #
    if options.skip_node_check:
        print "Skipping checks for node availability."
    else:
        gi = group.GroupInfo()
        down_node_ids = gi.getDownNodeList()
        down_node_lnns = map(gi.getLnn, down_node_ids)
        if down_node_lnns:
            for nl in down_node_lnns:
                print "WARNING: node %s is down, skipping..." % nl
                options.isi_nodes_args += ['-x', str(nl)]

    cluster.nodes = runIsiNodes(options.isi_nodes_args)
    if not cluster.nodes:
        error("Invalid node list.")

    #
    # Save the invoked command line to the tarball
    #
    saveCommandLine()

    #
    # Get "isi firmware status" output. That command accepts the same
    # set of isi_nodes args, so running on the specified nodes is easy.
    #
    # NOTE: potentially unsafe, commented out until resolution of bug 43316.
    #       Fixed by bug 82479
    runIsiFirmwareStatus(options.isi_nodes_args)

    #
    # Convert -f filenames to UTILITIES commands; Add to UTILITIES
    # Convert gather_script/_expr to UTILITIES commands; Add to UTILITIES
    #
    extra_includes = []
    extra_includes.extend(addExtraFiles(options.extra_file_list))
    if not is_compliance():
        extra_includes.extend(addExtraScripts(options.gather_script))
        extra_includes.extend(addExtraExprs(options.gather_expr))
        extra_includes.extend(addExtraLocalExprs(options.gather_expr_local))

    #
    # Now create include_list using the final updated UTILITIES.
    # NOTE this step MUST follow the above AddExtraFoobar calls.
    # We also need to add any new extra_includes to the include_list
    # (although only if includes isn't "all").
    #
    if "all" in options.includes:
        include_list = UTILITIES.keys()
    else:
        include_list = options.includes
    include_list.extend(extra_includes)

    #
    # write the temp script that contains the instructions for information
    # gathering.  we're going to write this in TMP_SUPPORT_DIR,
    # so we need to 1) see if that dir already exists, and 2) create it
    # (and any parents) if it doesn't exist.
    #
    # note that the gathered information from the nodes is temporary,
    # and will be removed at the end of the script.  however, the final
    # package file (stored in PKG_SUPPORT_DIR, or specified tarfile path)
    # will be uniquely preserved.
    #
    # Note we need to make these directories on all nodes using ssh,
    # if we're not using /ifs.
    #
    # I'm purposefully not honoring the -n option when creating the node
    # directories -- all directories will be created -- on all nodes.
    #
    # NOTE: The paths created here for the nodes MUST match those
    #       utilized in the generate_scripts routine.
    dirs = [SUPPORT_DIR, TMP_SUPPORT_DIR, PKG_SUPPORT_DIR,
            LOCAL_TMP_SUPPORT_DIR]
    for n, a in cluster.nodes:
        nodedir = TMP_SUPPORT_DIR + n
        dirs.append(nodedir)
    count = makeDirs(dirs, options.use_ifs, options.local_only)
    if count:
        error("FAILED to make required directories on %d nodes." % count)


    try:
        last_full_gather = int(os.stat(FULLGATHER_FILE).st_mtime)
    except:
        last_full_gather = 0

    if options.incremental:
        if os.path.exists(INCREMENTAL_FILE):
            # failures here may reset the incremental sequence to 0
            try:
                incfile = open(INCREMENTAL_FILE, 'r')
                incdata = incfile.read()
                incfile.close()

                # this file may have placeholder data from previous
                # versions.
                if incdata.isdigit():
                    incremental_sequence = int(incdata)
            except OSError:
                pass

            incremental_sequence += 1
        else:
            sys.stderr.write("WARNING: No previous run recorded. ")
            sys.stderr.write("Falling back to non-incremental gather.\n")
            options.incremental = False

    #
    # if --cores/--dumps specified explicitly than include cores/dumps in the package
    #
    if options.cores:
        include_list.append('cores')
    if options.dumps:
        include_list.append('isi_hangdump')

    #
    # build the gather script, based on the known information gathering
    # utilities, and the utilities the user is wishing to include
    #
    run_utils = buildRunUtils(include_list)
    if len(run_utils) == 0:
        error("No commands or info found to gather")

    generate_scripts(run_utils)

    #
    # run everything.
    #
    start_time=int(time.time())
    exit_status = 0

    print
    print "This may take several minutes.  Please do not interrupt the script."
    print

    activityIndicator.start()
    log.write("Gather In Progress on local node. Running.")
    task = "Running gather script on local node"
    (exit, output) = procs.get_cmd_output("bash " + LOCAL_GATHER_SCRIPT)
    if exit:
        print "ERROR %s" % task
        log.write("Gather Failed. %s" % output)
        for line in output:
            print line

    #
    # Save package xml metadata
    #
    pkginfo = {
        "gather_date": start_time,
        "last_full":   last_full_gather,
        "sequence":    incremental_sequence,
    }
    # if this generation fails, we should continue, but support
    # would like to see the traceback, and requested that we just
    # dump it into the file where the xml data should be.
    try:
        savePackageInfo(pkginfo, PACKAGE_INFO)
    except Exception, e:
        try:
            traceback.print_exc(file=open(PACKAGE_INFO, 'a'))
        except:
            pass

    try:
        saveCelogEvents(CELOG_EVENTS_INFO)
    except Exception, e:
        try:
            traceback.print_exc(file=open(CELOG_EVENTS_INFO, 'a'))
        except:
            pass

    post_args = []

    if is_compliance():
        post_args += ['sudo']

    post_args += ['isi_gather_info_slave']
    post_args += ['-j', '%d-%d' % (array.getLocalId(), os.getpid())]
    if options.verbose:
        post_args += ['--verbose']

    if cluster.user is not None:
        post_args += ['-u', cluster.user]

    if cluster.password is not None:
        post_args += ['-p', cluster.password]

    if not options.local_only:
        #
        # Report any problems up-front.
        #
        task = "Running script on all cluster nodes"
        verbose("%s..." % task)
        exit_status, error_count, exit_results = runCommandOnNodes(post_args)
        if error_count:
            exit_status = 1
            print "ERROR %s (on %d nodes)." % (task, error_count)
            log.write("Gather Failed. %s (%s)" % (exit_results, error_count))

        #
        # If we're not using /ifs, we need to collect gather script
        # results from all nodes under test.
        #

        if not options.use_ifs:
            task = "Collecting script results from all cluster nodes"
            verbose("%s..." % task)
            if copyResultsFromNodes():
                exit_status = 1
                print "ERROR %s" % task
                log.write("Gather Failed. (%s)"  % task)

    #
    # run any analysis scripts
    # NOTE We're not dealing with options.analysis.expr just now
    #
    task = "Analyzing script results from all cluster nodes"
    if len(options.analysis_script):
        verbose("%s..." % task)
        if analyzeResultsFromNodes():
            exit_status = 1
            print "ERROR %s" % task
            log.write("Gather Failed. (%s)" % task)

    #
    # put everything into one tarball
    #
    log.write("Gather Succeeded.")
    print "Information gathering completed... creating compressed package."
    pkg_path_name = makePkgName()

    try:
        tar_command = "/usr/bin/tar czf %s . 2>&1" % pkg_path_name
        os.chdir(TMP_SUPPORT_DIR)
        (exit, output) = procs.get_cmd_output(tar_command)
        if exit:
            print "ERROR tar returned %d" % (exit >> 8),
            log.write("Gather Failed")
            if output:
                print ", output follows:"
                for line in output:
                    print line
            else:
                print ""
            error("Could not create compressed package.")
            log.write("Gather Failed. Could not create compressed package.")
    except OSError, (errno, strerror):
        error("FAILED system call (tar): %s" % (os.strerror(errno)))
        log.write("Gather Failed %s" % (os.strerror(errno)))

    if options.symlinkdir:
        (head, tail) = os.path.split(pkg_path_name)
        symlink_path = os.path.join(options.symlinkdir, tail)
        try:
            os.symlink(pkg_path_name, symlink_path)
        except OSError, e:
            errstr = "Failed to symlink %s to %s: %s" % (pkg_path_name,
                    symlink_path, e)
            error(errstr)
            log.write(errstr)


    activityIndicator.join()
    print "Packaging complete..."
    print "Package: %s" % pkg_path_name
    log.write("Package: %s" % pkg_path_name)

    #
    # If we don't have the exit_status set, then the log gathered
    # completely, and we should create the touchfile for incremental log
    # gathers
    #
    if not exit_status:
        try:
            incfile = open(INCREMENTAL_FILE, 'w')
            incfile.write(str(incremental_sequence))
            incfile.close()
        except:
            sys.stderr.write('WARNING: Incremental run data not recorded.\n')
        try:
            os.utime(INCREMENTAL_FILE, (start_time, start_time))
        except:
            sys.stderr.write('WARNING: Incremental run data update failed.\n')
            log.write("Gather Failed. WARNING: Incremental run data update failed.")

        if not options.incremental:
            if not os.path.exists(FULLGATHER_FILE):
                try:
                    fullgather = open(FULLGATHER_FILE, 'w')
                    fullgather.write('This space intentionally left blank.\n')
                    fullgather.close()
                except:
                    sys.stderr.write('WARNING: Full Gather run data not recorded.\n')

            try:
                os.utime(FULLGATHER_FILE, (start_time, start_time))
            except Exception, e:
                sys.stderr.write('WARNING: Full Gather run data update failed. %s\n' % str(e))
                log.write("Gather Failed. WARNING: Full Gather run data update failed.")

    if options.upload_package:
        print "Uploading in progress.  If problems are encountered during the"
        print "upload process, the package will need to be sent manually."
        uploadPackage(pkg_path_name)
    else:
        print "No Upload method selected. Package not uploaded."
        log.write("No Upload method selected. Package not uploaded.")

    print "Cleaning up temporary data...",
    cleanupGatherRunFile()
    if clean_tmp_dir(TMP_SUPPORT_DIR) == 0:
        print "done."

    verbose("\nCleaning up core/dumps")
    if options.clean_cores:
        if clean_cores(True) == 0:
            print "core clean done."
    if options.clean_dumps:
        if clean_cores(False) == 0:
            print "dump clean done."

    log.join()
    sys.exit(exit_status)

def runIsiNodes(args):
    isi_nodes_specifiers = '%{id} %{internal} %{node} %{lnn}'
    nodes = []

    # Add a column label line
    (exit, output) = procs.get_cmd_output('echo %s > %s' % \
                                          (isi_nodes_specifiers, \
                                           ISI_NODES_OUTPUT))
    if exit:
        if len(output):
            print output[0]
        else:
            print "Error writing file", ISI_NODES_OUTPUT
        return None

    (exit, output) = procs.get_cmd_output('isi_nodes %s %s >> %s' % \
                                          (string.join(options.isi_nodes_args),\
                                           isi_nodes_specifiers, \
                                           ISI_NODES_OUTPUT))
    if exit:
        if len(output):
            print output[0]
        else:
            print "Error running isi_nodes."
        return None

    isi_nodes_file = open(ISI_NODES_OUTPUT, 'r')

    # Skip column label line
    isi_nodes_file.readline()
    for line in isi_nodes_file:
        toks = line.split()
        # tuple of %{node} and %{ip_address} from isi_nodes
        nodes.append((toks[2], toks[1]))

    isi_nodes_file.close()

    return nodes

def saveCommandLine():
    try:
        mode = os.O_RDWR|os.O_CREAT|os.O_TRUNC
        fd = os.open(SAVED_COMMAND_LINE, mode, 0666)
        os.write(fd, " ".join(sys.argv) + "\n")
        os.close(fd)
    except OSError, e:
        error("Failed saving command-line output")

def savePackageInfo(pkginfo, fname):
    # this saves to the package information about the package

    contents = []
    if options.no_exempt_logs:
        contents.append('none')
    elif options.incremental:
        contents.append('incremental')
    else:
        contents.append('full')

    for group in options.group:
        contents.append("group:%s" % group)

    # Bug 167340 - switch to git attr ident
    # Use \x24 instead of $ to avoid git interpolation
    revision = re.match(r"\x24Id: ([\da-f]+) \x24", REVISION).group(1)

    # generate our xml
    pkg = etree.Element("package")
    pkg.attrib["version"] = "1"

    etree.SubElement(pkg, "command").text     = " ".join(sys.argv)
    etree.SubElement(pkg, "source_lnn").text  = str(getLocalLNN())
    etree.SubElement(pkg, "contents").text    = ",".join(contents)
    etree.SubElement(pkg, "gather_date").text = str(pkginfo["gather_date"])
    etree.SubElement(pkg, "revision").text    = revision

    logs = etree.Element("logs")
    etree.SubElement(logs, "last_full").text  = str(pkginfo["last_full"])
    etree.SubElement(logs, "sequence").text   = str(pkginfo["sequence"])
    pkg.append(logs)

    f = open(fname, 'w')
    f.write(etree.tostring(pkg))
    f.close()

def saveCelogEvents(fname):
    # this saves to the package identifying information about the
    # cluster.  all the information we want is directly generated
    # by celog events, so we're just reusing it.

    # XXX this section is going to change to be more
    # self contained in the celog libraries.
    cfg = isi.celog.celog_config_api.CelogConfigAPI()
    custcfg = cfg.get_contact_settings()

    guid = arrcfg.config_load().guid

    ev = isi.celog.attachments.AllEvents([])
    ev.custcfg = custcfg
    ev.guid = guid
    evxml = ev.get_evxml(ev.events)

    f = open(fname, 'w')
    f.write(etree.tostring(evxml))
    f.close()


def runIsiFirmwareStatus(args):
    cmd = ('/usr/bin/isi firmware status --verbose %s > %s' % \
           (' '.join(args), ISI_FIRMWARE_OUTPUT))
    if is_compliance():
        cmd = 'sudo ' + cmd
    (exit, output) = procs.get_cmd_output(cmd)
    if exit:
        print 'Error running "isi firmware status"'
        for line in output:
            print line

def waitForPid(pid):
    while True:
        pid, status = os.waitpid(pid, 0)
        if os.WIFEXITED(status):
            return os.WEXITSTATUS(status)
        elif os.WIFSIGNALED(status):
            return 1

def waitForChildren(children):
    #
    # NOTE using max(exit_results) could miss potential negative returns,
    # masked by (greater than) 0 success results.  This code will still
    # get max, ignoring all 0s, returning negatives if caught and "max".
    #
    exit_results = [waitForPid(child) for child in children]
    exit_errors = filter(lambda x: x!=0, exit_results)
    error_count = len(exit_errors)
    exit_status = error_count and reduce(max, exit_errors) or 0
    return(exit_status, error_count, exit_results)

def hasQuorum():
    try:
        fp = os.popen("sysctl -n efs.gmp.has_quorum", 'r')
    except OSError, (errno, strerror):
        error("popen failed on efs.gmp.has_quorum: %s" % os.strerror(errno))

    quorum = fp.read()
    fp.close()
    quorum = quorum.strip('\n')

    if quorum == "1":
        return True
    elif quorum == "0":
        return False
    else:
        print "Unknown return from efs.gmp.has_quorum: %s" % quorum
        return False

def getLocalLNN():
    try:
        fp = os.popen("/usr/bin/isi_nodes -L %{lnn}", 'r')
        isi_nodes = fp.read()
        fp.close()
        lnn = int(isi_nodes.strip('\n'))
    except:
        lnn = -1

    return lnn

def makeDir(path):
    if not os.path.exists(path):
        try:
            os.makedirs(path, 0775)
        except OSError, (errno, strerror):
            error("couldn't make %s directory: %s" % \
                (path, os.strerror(errno)))

def getHostname():
    try:
        f = os.popen("/bin/hostname", "r")
        host_name = f.readline()
    except OSError, (errno, strerror):
        error("error running hostname: %s" % os.strerror(errno))
    f.close()

    host_name = host_name.strip()
    names     = host_name.split('-')

    # this code does :
    # host_name      len(names) final names:
    # something      1          something
    # something-2    2          something
    # some-thing-2   3          some-thing
    # some-th-ing-2  4          some-th-ing
    if len(names) > 1:
        names.pop()
    cluster_name = '-'.join(names)

    return cluster_name, host_name

def makePkgName():
    if options.tarfile != None:
        pkg_path_name = fixTarfileName(options.tarfile)
        if pkg_path_name != None:
            return(pkg_path_name)

    time_string = time.strftime("%Y%m%d-%H%M%S")
    pkg_name = "IsilonLogs-%s-%s.tgz" % (cluster.name, time_string)
    pkg_path_name = os.path.join(PKG_SUPPORT_DIR, pkg_name)

    return(pkg_path_name)

def makeDirs(dirs, use_ifs=False, local_only=False):
    '''Returns count of nodes that failed mkdir command.'''
    if use_ifs or cluster is None or cluster.nodes is None or local_only:
        for dir in dirs:
            makeDir(dir)
        return(0)
    cmd_args = []
    if is_compliance():
        cmd_args.append('sudo')
    cmd_args += ['isi_gather_info_slave', '-s']
    exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
    if error_count > 0:
        return (error_count)

    cmd_args = ['mkdir','-p'] + dirs
    exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
    return(error_count)

def listGroups():
    print "\nKnown groups (included with the --group option): \n"
    groups = GROUPS.keys()
    groups.sort()
    for g in groups:
        print "  %s" % g
    print

def listUtils():
    print
    print "Known utilities (included with the -i option): \n"
    utils = UTILITIES.keys()
    utils.sort()
    for u in utils:
        if u not in EXEMPT:
            print "  %s" % u
    print

#
# this routine is simply here to print the warning message, and alert
# the user they munged the name.  i don't go through the pain of
# removing the unknown name from the include array, since it won't
# hurt anything being in there (since the keys of the UTILITIES dict
# is what's searched).
#
def pruneUtils(requested):
    ulist = UTILITIES.keys()
    nlist = []
    for r in requested:
        if not r in ulist:
            print "WARNING: unknown utility (%s), ignored..." % r
        else:
            nlist.append(r)

    return(nlist)

#
# An internal correctness check to verify that everything in GROUPS and
# EXEMPT actually exists.
#
def validateUtils():
    util_errors = 0
    for (group, utils) in GROUPS.items():
        for util in utils:
            if not UTILITIES.has_key(util):
                util_errors += 1
                print "Unknown utility '%s' in group '%s'!" % (util, group)
    for util in EXEMPT:
        if not UTILITIES.has_key(util):
            print "Unknown utility '%s' in default group!" % util
    if util_errors:
        print "%d utility errors." % util_errors
        sys.exit(1)

#
# the logic: anything (valid) in the include list automatically means
# all other utilities are excluded (except for things in EXEMPT).
# (unless user specified '--nologs', which skips EXEMPT)
#
def buildRunUtils(ilist):
    if ilist != None:
        list = pruneUtils(ilist)
    else:
        list = []
    if options.group_utils:
        list.extend(options.group_utils)
    if options.no_exempt_logs == False:
        list.extend(EXEMPT)
    return(list)

#
# Given a user-supplied tarfile, fix it to ensure:
# o If full path specified, use that path
# o If no full path given, use original PKG_SUPPORT_DIR path
# o Check extension: if usual tarfile extension, make sure we
#   don't "double up" with additional .tgz; otherwise keep intact.
# o Also check for special case '/' (root) specified
# Returns None for any problems (such as '/' case)
#
def fixTarfileName(tarfile):
    tarfile = os.path.normpath(tarfile.strip())
    #
    # Note this verifies we were not given only '/'
    #
    if len(tarfile) and not (tarfile[0]=='/' and len(tarfile)==1):
        #
        # Is it a full path or not? If not, supply default path.
        #
        if tarfile[0] != '/':
            tarfile = PKG_SUPPORT_DIR + tarfile
        #
        # Does it have an extension already?
        #
        root,ext = os.path.splitext(tarfile)
        if ext in ['.tar','.tgz']:
            tarfile = root
        elif ext in ['.gz']:
            root2,ext2 = os.path.splitext(root)
            if ext2 in ['.tar']:
                tarfile = root2
            else:
                tarfile = root
        else:
            #
            # Note we'll just keep original extensions...
            #
            pass
        tarfile = tarfile + ".tgz"
        return(tarfile)
    else:
        return(None)

def getPathFile(pathfile):
    # NOTE we need this rstrip mod for reliable dirname/basename!
    x = pathfile.rstrip('/')
    return(os.path.dirname(x), os.path.basename(x))

def getUtilityGatherCmd(name, path, base):
    #
    # Avoid gathering duplicate data from all the nodes.
    # If path starts with /ifs, than gather data from only one node (local node)
    # e.g. path=/ifs/data or path=/ifs
    #
    if not options.local_only and len(path) \
            and ((path[0:4] == '/ifs') or (path == '/' and (name == 'ifs'))):
        c = ('local_tar', "cd %s" % (path), "%s.tar %s" % (name, base))
    else:
        c = ('tar', "cd %s" % (path), "%s.tar %s" % (name, base))
    return(c)

#
# 'cmd' better be executable - either fullpath or in PATH
#
def getUtilityExecuteCmd(name, cmd, args):
    c = ('exe', "%s %s" % (cmd, args), "%s" % (name))
    return(c)

def getUtilityExecuteLocalCmd(name, cmd, args):
    c = ('local_exe', "%s %s" % (cmd, args), "%s" % (name))
    return(c)

def makeUniqueDictKeyName(name, dict):
    new_name, count = name, 0
    while new_name in dict.keys():
        count += 1
        new_name = '%s-%d' % (name, count)
    return new_name

#
# Use a dict and we get unique non-repeated entries!
# (Since the key name is uniquely determined from given extra_list)
# Be sure to rstrip '/' from any dirs/cmds to get proper 'basename'.
# Be sure to lstrip '/' from any dirs/cmds/exprs to start names without
# extra baggage.  ===> rstrip + lstrip = just strip
# Get unique 'name' from stripping away nonalnum, or replacing.
#
def getExtraDict(extra_list, extra_entry_func):
    #
    # This is really just for the expressions:
    # Some additional common expression chars to replace
    # (by including '-' in this list, we'll reduce multi -'s)
    #
    replace_list = '\\|/.,;: <>-!@#$%&*()=+' + "'" + '"'
    retain_list = '_'

    extra = {}
    for e in extra_list:
        # use consistent strip of '/', see comments above
        old, new = list(e.strip('/')), []
        last_replace = False
        for o in old:
            if o.isspace() or o in replace_list:
                if not last_replace:
                    new.append('-')
                    last_replace = True
            elif o.isalnum() or o in retain_list:
                new.append(o)
                last_replace = False
        name = ''.join(new)
        name = makeUniqueDictKeyName(name,extra)
        extra[name] = extra_entry_func(e)
    return(extra)

def addExtraFiles(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        extra_dict = getExtraDict(extra_list, lambda e: getPathFile(e))
        for name,(path,base) in extra_dict.iteritems():
            # Don't gather /ifs/data/Isilon_Support
            if (os.path.join(path, base) in ('/ifs/data/Isilon_Support',
                '/ifs/data/Isilon_Support/pkg', '/ifs/data/Isilon_Support/tmp')):
                print "WARNING: ignored path %s/%s" % (path, base)
                continue
            UTILITIES[name] = getUtilityGatherCmd(name,path,base)
            extra_includes.append(name)
    return(extra_includes)

#
# NOTE that simply adding these scripts to the include list isn't sufficient;
# we need to ensure they exist on the nodes, or are copied there if necessary.
# However, it is SIMPLER to leave this requirement up to the user or calling
# script. Perhaps isi_for_array or isi_scp.
#
def addExtraScripts(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        #
        # Future enhancement possibility:
        # Do we want user-supplied options, per-script?
        # will quotes work with getopt.getopt?
        # i.e. -s "scriptname --opt1 opt2" -otheroption ...
        # ==> (cmd,args) = (scriptname, "--opt1 opt2")
        #
        extra_dict = getExtraDict(extra_list, lambda e: (e, ''))
        for name,(cmd,args) in extra_dict.iteritems():
            UTILITIES[name] = getUtilityExecuteCmd(name,cmd,args)
            extra_includes.append(name)
    return(extra_includes)

def addExtraExprs(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        # leave e intact! (in the returned extra dict cmd)
        extra_dict = getExtraDict(extra_list, lambda e: (e, ''))
        for name,(cmd,args) in extra_dict.iteritems():
            UTILITIES[name] = getUtilityExecuteCmd(name,cmd,args)
            extra_includes.append(name)
    return(extra_includes)

def addExtraLocalExprs(extra_list):
    global UTILITIES
    extra_includes = []
    if extra_list != None and len(extra_list):
        extra_dict = getExtraDict(extra_list, lambda e: (e, ''))
        for name,(cmd,args) in extra_dict.iteritems():
            UTILITIES[name] = getUtilityExecuteLocalCmd(name,cmd,args)
            extra_includes.append(name)
    return(extra_includes)

def copyScriptToNodes(script):
    result = 0
    for node_name, node_address in cluster.nodes:
        dest = "%s:%s" % (node_name, script)
        scp_error = isi_scp_main(script, dest, cluster.user, cluster.password)
        if scp_error:
            error('Error copying script to %s' % node_name)
            # Note error is sys.exit... we currently won't actually return
            result += 1
    return(result)

#
# Use -r to recursively grab entire remote directory structure
# Keep in mind: GATHER_SCRIPT = TMP_SUPPORT_DIR + '_gather'
# So all the results are in TMP_SUPPORT_DIR, and in fact the
# commands executed spit results into TMP_SUPPORT_DIR$HOST/something
# So essentially we want to grab (scp -r) TMP_SUPPORT_DIR from
# each node, which would (if properly done at exec time) only have
# a node-unique $HOST dir in existence that would be copied to
# the main node, not interfering with any other node results.
#
# Note that using scp -r, we have to be careful:
# (1) we can't scp from remote:/path/tmp/ to local:/path/tmp/
#     because you end up with results in: local:/path/tmp/tmp/...
# (2) above problem is dealt with by using parent dir, i.e.,
#     scp from remote:/path/tmp to local:/path/
# (3) HOWEVER, we need to make sure we never do this when
#     remote host == local host (to self), to avoid recursion problems
#
def copyResultsFromNodes():
    result = 0
    dest, dirname = getPathFile(TMP_SUPPORT_DIR)
    for node_name, node_address in cluster.nodes:
        #
        # Don't copy from self to self:
        #
        if node_name == cluster.hostname:
            continue
        src = "%s:%s" % (node_name, TMP_SUPPORT_DIR)
        scp_error = isi_scp_main(src, dest, cluster.user, cluster.password,
                                 options=['-r'])
        if scp_error:
            error('Error copying results file from %s' % node_name)
            # Note error is sys.exit... we currently won't actually return
            result += 1
    return(result)

#
# Need to pass two arguments to script:
#    arg 1: full directory path location of results
#    arg 2: cluster_name
# With these two args, any script should be able to parse the results dir,
# find appropriate node subdirs (based on hostname). It is up to the script
# to determine what is there... It should not assume each dir is a results
# dir, nor should it make any assumptions about files in the top dir --
# These may be results of other analysis scripts that have previously run.
# The scripts can put any output or output logs into this directory --
# Though beware of overwriting any existing files!
# After all analysis scripts have run, everything in this top dir down will
# be tarred up into the final tarball.
#
def analyzeResultsFromNodes():
    results = 0
    results_dir = TMP_SUPPORT_DIR
    script_output = {}
    for script in options.analysis_script:
        verbose("\nRunning: %s" % script)
        name = script.strip('/').replace('/','-').replace('.','-')
        name = makeUniqueDictKeyName(name,script_output)
        script_output[name] = "%s%s.out" % (results_dir, name)
        command = "%s %s %s" % (script, results_dir, cluster.name)
        (exit, output) = procs.get_cmd_output(command)
        if output and len(output):
            try:
                fd = os.open(script_output[name],
                             os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0666)
                for line in output:
                    os.write(fd, "%s\n" % line)
                    #
                    # Changed from original:  verbose(line)
                    # We want this output to be seen to report any analysis
                    # output, always.
                    #
                    print line
                os.close(fd)
            except OSError, (errno, strerror):
                results += 1
                print "FAILED %s: %s" % (script, os.strerror(errno))
        if exit:
            results += 1
            print "FAILED %s: returned %d" % (script, exit >> 8)
    return(results)

def execCommandOnNodes(post_args):
    children = []
    pre_args = ['isi_rdo']
    for node_name, node_address in cluster.nodes:
        child = os.fork()
        if not child:
            if cluster.user != None:
                pre_args += ['-u', cluster.user]
            if cluster.password != None:
                temp_file = tempfile.TemporaryFile('w+b', -1, 'isi_for_array')
                temp_file.write(cluster.password)
                temp_file.flush()
                temp_file.seek(0)
                #
                # as of Python 2.3, TemporaryFile fd's are not inherited
                # by child procs so dup it for the exec below.
                #
                dupfd = os.dup(temp_file.fileno())
                temp_file.close()
                pre_args += ['-q', '/dev/fd/%d' % dupfd]
            pre_args += [node_address]
            os.execvp('isi_rdo', pre_args + post_args)
        else:
            children.append(child)
    return children

def runCommandOnNodes(cmd_args):
    '''Returns tuple (exit_status, error_count, exit_results):
       exit_status : max non-zero exit_result value
       error_count : number of non-zero exit status in exit_results
       exit_results: array of per-child exit status
    '''
    children = execCommandOnNodes(cmd_args)
    return waitForChildren(children)

def isi_scp_main(file1, file2, user=None, password=None, options=None):
    '''isi_scp [OPTION]... [[USER@]HOST1:]FILE1 [[USER@]HOST2:]FILE2'''
    cmd_args = ['isi_scp']
    children = []
    child = os.fork()
    if not child:
        if user != None:
            cmd_args += ['--user', user]
        if password != None:
            # change to isi_scp pre later? for association?
            temp_file = tempfile.TemporaryFile('w+b', -1, 'isi_for_array')
            temp_file.write(cluster.password)
            temp_file.flush()
            temp_file.seek(0)
            #
            # as of Python 2.3, TemporaryFile fd's are not inherited
            # by child procs so dup it for the exec below.
            #
            dupfd = os.dup(temp_file.fileno())
            temp_file.close()
            cmd_args += ['-q', '/dev/fd/%d' % dupfd]
        if options:
            cmd_args += reduce(lambda a,b: a+['-o']+['%s' % b], options, [])
        cmd_args += [ file1, file2 ]
        os.execvp('isi_scp', cmd_args)
    else:
        children.append(child)
    exit_status, error_count, exit_results = waitForChildren(children)
    return exit_status

# this routine is responsible for uploading the package to isilon.
# it will try any supported access methods (currently ftp,http,smtp), or
# as specified on the command line.
def uploadPackage(package_path):
    global options
    global log
    uploaded = False

    #first SMTP
    if options.email_enable:
        if(uploadPackageEmail(package_path)):
            uploaded = True
        else:
            uploaded = False

    #then http
    if options.http_enable and options.http_host and not uploaded:
        try:
            client = MyHttp(options.http_host, options.http_path, \
                            options.http_proxy_host, options.http_proxy_port)
            msg = "Upload in Progress. (HTTP). Filename: %s" % \
                os.path.basename(package_path)
            print msg
            log.write(msg)

            package = file(package_path,'r')
            (exit, ftpError) = client.storeFile(package)
            if exit:
                uploaded = False
                raise Exception(ftpError)
            else:
                uploaded = True
                msg ="Upload Succeeded. (HTTP). Filename: %s" % \
                          os.path.basename(package_path)
                print msg
                log.write(msg)
        except Exception, e:
            msg = "Upload Failed to HTTP host: [%s]. Error: %s" % \
                (options.http_host, str(e))
            print msg
            log.write(msg)

    tree = gconfig.Tree("remote-support")
    base = gconfig.open(tree)
    ctx = base.ctx_new()
    connectemc_cfg = ctx.read("connectemc")

    if options.esrs and connectemc_cfg["enabled"]:
        esrs_host = connectemc_cfg["esrs_host"]

        for gateway in ("primary_esrs_gateway", "secondary_esrs_gateway"):
            if connectemc_cfg[gateway] and not uploaded:
                try:
                    error = send_file_to_esrs_via_curl(esrs_host,
                            connectemc_cfg[gateway], "8118",
                            "/r/mft/incoming", package_path)

                    # curl returns error code 0 if connected
                    if error == 0:
                        # uploaded successfully
                        uploaded = True
                except Exception as e:
                    msg = "Upload Failed to HTTP host: [%s]. Error: %s" % \
                            (esrs_host, str(e))
                    print msg
                    traceback.print_exc()
                    log.write(msg)

    #then ftp
    if options.ftp_enable and not uploaded:
        if not uploaded and options.try_passive_ftp:
            print "Trying Passive FTP..."
            log.write("Upload in Progress. (FTP - Passive Upload).")
            try:
                uploaded = uploadPackageFTP(package_path, pasv=True)
            except ftplib.all_errors, e:
                msg = "Upload Failed to FTP host: [%s]. (Passive Upload)."\
                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)
            except Exception, e:
                msg = "Upload Failed to FTP host: [%s]. (Passive Upload)." \
                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)

            if uploaded:
                msg = "Uploaded Succeeded (FTP - Passive). File %s" % \
                    os.path.basename(package_path)
                print msg
                log.write(msg)

        if not uploaded and options.try_active_ftp:
            print "Trying Active FTP..."
            log.write("Upload in Progress. (FTP - Active Upload).")
            try:
                uploaded = uploadPackageFTP(package_path, pasv=False)
            except ftplib.all_errors, e:
                msg = "Upload Failed to FTP host: [%s]. (Active Upload)."\
                                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)
            except Exception, e:
                msg = "Upload Failed to FTP host: [%s]. (Active Upload)."\
                " Error: %s" % (options.ftp_host, str(e))
                print msg
                log.write(msg)

            if uploaded:
                msg = "Upload Succeeded (FTP - Active). File %s" % \
                                os.path.basename(package_path)
                print msg
                log.write(msg)

    if not uploaded:
        msg = "Package upload failed with all protocols please upload it manually."
        print(msg)
        log.write(msg)
    return uploaded

def uploadPackageEmail(package_path):
    global log
    global options

    email_to = []

    for address in options.email_addresses.split(','):
        if EMAIL_REGEX.match(address.lstrip()):
            email_to.append(address.strip())
        else:
            msg ='%s is not valid email address. Skipped.' % address
            log.write(msg)
            print msg
    if email_to is []:
        msg ='Upload Failed. (SMTP).  No valid email address was specified.'
        log.write(msg)
        print msg
        return False

    from_address = options.email_from
    if not from_address:
        from_address = None
    elif not EMAIL_REGEX.match(from_address.lstrip()):
        msg = "%s is not valid email address. Continuing without sender " \
              "email address." % from_address
        log.write(msg)
        print msg
        from_address = None

    subject = options.email_subject
    if not subject:
        subject = 'Auto generated e-mail by isi_gather_info on %s' % \
            cluster.hostname

    text = options.email_body
    if not text:
        text =  'This is auto generated e-mail by isi_gather_info on %s' % \
            cluster.hostname

    text += generateEmailBody()

    # check filesize of gather file.
    filesize = os.path.getsize(package_path)
    if filesize > 5242880 and not options.skip_size_check:
        msg = "File size is %d KBytes. It's very big for e-mail.\n" \
              "Sending each gather file with -n option,  " \
              "--varlog_recent , --incremental, -x history are " \
              "useful to reduce file size. If you want to skip " \
              "size check, use --skip-size-check option. \n" % \
            (filesize / 1024)
        print msg
        log.write(msg)
        return False
    attachments = [EmailAttachmentFromFile(filepath=package_path,
            mime_type="application/octet-stream")]
    try:
        Emailer.send_email(to_addresses = email_to, message = text,
                from_address = from_address, subject = subject,
                attachments = attachments)
        send_result = True

        msg ="Upload Succeeded to mail address: %s  Filename: %s" % \
            (email_to, os.path.basename(package_path))
        log.write(msg)
        print msg
    except EmailError, e:
        send_result = False
        log.write(str(e))
        print str(e)
    return send_result

def forArrayCmd(nodeOps, args):
    proxy = rpc.RPCDaemonClusterProxy(nodeOps,
            'isi.app.lib.procs.RunCmdOutput')
    result = proxy.run(args, raw = True)
    sorted_keys = result.keys()
    sorted_keys.sort(lambda x, y: cmp(x.lnn(), y.lnn()))
    msglist = []
    for key in sorted_keys:
        if result[key][1]:
            msglist.append('%s:\n' % key.name())
            msglist.append(str(result[key][1]))
    return ''.join(msglist)


def generateEmailBody():
    global log

    # get up node list.
    nodeOps = array.NodeOps()
    gi = group.GroupInfo()
    down_node_ids = gi.getDownNodeList()
    if down_node_ids:
        for ni in down_node_ids:
            nodeOps.exclude(str(gi.getLnn(ni)))

    outputlist = ['\n\n']
    for (cmd, flag) in MAIL_COMMAND:
        if flag == 'node':
            outputlist.append('\n\n%s on %s:\n' % (cmd, cluster.hostname))
            try:
                (exit, output) = procs.get_cmd_output(cmd, raw=True)
                outputlist.append(output)
            except Exception, e:
                msg =  '%s returned error: %s' % (cmd, str(e))
                print msg
                log.write(msg)
                pass
        else:
            outputlist.append('\n\n"%s" on all up nodes:\n' % cmd)
            try:
                outputlist.append(forArrayCmd(nodeOps, cmd))
            except Exception, e:
                msg = 'Execution of "%s" returned error: %s' % \
                    (cmd, str(e))
                print msg
                log.write(msg)
                pass
    outputlist.append('\n')
    text = ''.join(outputlist)
    return text


def uploadPackageFTP(package_path, pasv=True):
    global options
    global log

    package_filename = os.path.basename(package_path)
    package = file(package_path, "r")

    client = MyFTP(options.ftp_host, options.ftp_user, options.ftp_pass, \
                   options.ftp_path, options.ftp_port, options.ftp_proxy_host, \
                   options.ftp_proxy_port)
    client.set_pasv(pasv)
    try:
        response = client.storbinary("STOR %s" % package_filename, package)
        if '226' in response:
            log.write("Upload Succeeded. File %s" % package_filename)
            return True
        else:
            msg = "Upload Failed to FTP host: [%s]. Response: %s" % (options.ftp_host, response)
            print msg
            log.write(msg)
            return False
        print       #flushes the sys.stdout buffer and start newline
        try:
            client.quit()
            client.close()
            package.close()
        except ftplib.all_errors:
            # ftp.isilon.com drops connections in a violent way
            # when client.quit() is called, which causes ftplib to
            # raise an exception.
            # Ignore any errors at this point, because we've
            # completed the upload if we've reached this far,
            # and we simply don't care about the rest.
            pass
            return True

    except Exception, e:
        msg = "Upload Failed to FTP host: [%s]. Response: %s" % (options.ftp_host, str(e))
        print msg
        log.write(msg)
        return False

def reUploadPackage(item):
    archivePath = PKG_SUPPORT_DIR#IFS_SUPPORT_DIR+_PKG_SUPPORT_DIR
    itempath = os.path.realpath(item)
    flock = MyLock()

    try:
        #is it a complete path?
        if os.path.exists(itempath):
            fname = itempath
            uploadPackage(fname)
        #is it a file? then attach archives path
        elif os.path.exists(os.path.join(archivePath,item)):
            fname = os.path.join(archivePath, item)
            uploadPackage(fname)
        else:
            flock.unlock()
            error("Log file: %s could not be found" % item)
    except Exception, e:
        error("Error Re-Uploading file %s" % str(e))
        flock.unlock()
    flock.unlock()

def verifyConnection():
    def sigTimerHandler(self, signum, frame):
        raise Exception ("Connection Verification Failed: Connection Timed Out.")

    global log
    import tarfile
    from cStringIO import StringIO

    log.write("Connection Verification in Progress")
    print "Connection Verification in Progress"

    ##create tar file###
    path = '/tmp/'
    fname = 'isiVerifyFile-' + time.strftime('%Y%m%d-%H%M%S') + '.tar.gz' #salt
    myTarFile = tarfile.open(path + fname, 'w:gz')
    try:
        try:
            data = "DEADBEEF"*100
            info = tarfile.TarInfo('garbagefile')
            info.size = len(data)
            myTarFile.addfile(info, StringIO(data))
        except Exception, e:
            print "Connection Verification (HTTP) Failed: Tarfile" \
            "creation error. %s" % str(e)
            log.write("Connection Verification (HTTP) Failed: Tarfile" \
            "creation error. %s" % str(e))
            sys.exit(0)
    finally:
        myTarFile.close()
   #/create tar file###

    #CAVEAT: can wreak havoc if multiple procs are in process
    signal.signal(signal.SIGALRM, sigTimerHandler)
    signal.alarm(300)       #5-minute timeout
    try:
        success = uploadPackage(path+fname)
        if success:
            print "Connection Verification Succeeded."
            log.write("Connection Verification Succeeded.")
        else:
            print "Connection Verification Failed."
            log.write("Connection Verification Failed.")
    except Exception, e:
        log.write("Connection Verification Failed. %s" % str(e))
        print "Connection Verification Failed. %s" % str(e)
        pass

    try:
        os.remove(path+fname)
    except:
        print "Error in removing temporary file: %s " % str(path+fname)
        log.write("Error in removing temporary file: %s " % str(path+fname))
        pass
    else:
        print "Temp verify file removed."

    sys.exit(0)
    return

def verbose(str):
    if options.verbose:
        print >> sys.stdout, str

def error(str):
    print >> sys.stderr, PROGNAME + ':', str
    sys.exit(1)

def clean_tmp_dir(dir=TMP_SUPPORT_DIR):
    '''Function cleans up all temporary data on all nodes.  Returns the number
        of failures returned from executing rm.'''
    error_count = 0
    cmd_args = ["/bin/rm", "-rf", dir]
    if options.use_ifs or options.local_only:
        cmd = " ".join(cmd_args)
        (exit, output) = procs.get_cmd_output(cmd)
        if exit:
            error_count += 1
            print "\nERROR cleaning up %s, output follows:\n%s" % (dir, \
                                                        "\n".join(output))
    else:
        exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
        if error_count:
            print "\nERROR cleaning up %s on %d nodes." % (dir, error_count)
    if error_count:
        print "You may wish to delete these files manually."
    return error_count

def clean_cores(isCore=True):
    '''Function cleans up all cores/dumps on all nodes.  Returns the number
        of failures returned from executing rm.'''
    error_count = 0
    if isCore:
        cmd_args = ["/bin/rm", "-rf", "/var/crash/*.core.gz"]
    else:
        cmd_args = ["/bin/rm", "-rf", "/var/crash/isi_hangdump*"]
    if options.use_ifs or options.local_only:
        cmd = " ".join(cmd_args)
        (exit, output) = procs.get_cmd_output(cmd)
        if exit:
            error_count += 1
            print "\nERROR cleaning up %s, output follows:\n%s" % (dir, \
                                                        "\n".join(output))
    else:
        exit_status, error_count, exit_results = runCommandOnNodes(cmd_args)
        if error_count:
            print "\nERROR cleaning up %s on %d nodes." % (dir, error_count)
    if error_count:
        print "You may wish to delete these files manually."
    return error_count

class MyHttp(httplib.HTTPConnection):
    """extends httplib.HTTPConnection with a storeFile that has a callback
        function. the callback function intercepts the status of the file
        transfer. a ProgressBar implementation that prints to stdout.
    """
    def __init__(self, host='', path='/put/', proxy='', proxyport=''):
        self.startTime = 0
        self.fileSize = 0
        self.uploaded = 0
        self.updateLogIntVal = 3                #interval to log to file
        self.updateTtyIntVal = 1                #interval to update progress
        self.lastLogTime = 0
        self.lastTtyTime = 0
        self.pbar = ''                          #ProgressBar
        self.enableLog = True
        global log
        self.log = log

        if proxy and proxyport:
            httplib.HTTPConnection.__init__(self, host=proxy, port=proxyport, \
                                            strict=0)
        else:
            httplib.HTTPConnection.__init__(self, host=host, port=80, strict=0)
        self.url = 'http://[%s]/%s/' % (host, path.strip('/'))

    def _prepHeaders(self, fp):
        """helper function that sets the appropriate headers
        """
        try:
            self.putrequest('PUT',  self.url+ os.path.basename(fp.name))
            self.putheader('User-Agent', 'isi_gather_info')
            self.putheader('Connection', 'keep-alive')
            self.putheader('Transfer-Encoding', 'chunked')
            self.putheader('Accept', '*/*')
            self.putheader('Expect', '100-continue')
            self.endheaders()
            return (0,'')
        except Exception, e:
            return (1, e)

    def _prepTransfer(self, fp):
        """helper function to get file/time data
        """
        self.fileSize = os.stat(fp.name).st_size
        self.startTime = time.time()
        self.pbar = ProgressBar(self.fileSize)

    def storeFile(self, fp, blocksize=8192):
        """stores the file object (fp) to the host.
            runs the callback function per 'blocksize' data transferred.
        """
        exit, error = self._prepHeaders(fp)
        if exit:
            return (exit, error)
        self._prepTransfer(fp)

        while True:
            buf = fp.read(blocksize)
            if not buf: break
            length = len(buf)
            self.send('%X\r\n' % length)
            self.send(buf + '\r\n')
            self.cbFunc(length)
        self.send('0\r\n\r\n')
        print #flush
        return (0,'')

    #callback func gets uploaded packet size
    def cbFunc(self, bufSize):
        now = time.time()
        self.uploaded += bufSize
        self.timeElapsed = now - self.startTime
        if sys.stdout.isatty() and \
        (now - self.lastTtyTime > self.updateTtyIntVal):
            self.lastTtyTime = now
            self.pbar.update(self.uploaded)

        if self.enableLog and \
        (now - self.lastLogTime > self.updateLogIntVal):
            msg = "Uploaded: %s  out of %s . Time Elapsed: %s" % \
            (format_capacity(self.uploaded), format_capacity(self.fileSize), \
             time.strftime("%H:%M:%S", time.gmtime(self.timeElapsed)))
            self.lastLogTime = now
            self.log.write(msg)


def send_file_to_esrs_via_curl(host, proxy, proxyport, uri, filepath):

    serial_number = isi.app.lib.cluster.DeviceInfo().serial_number()
    guid = arrcfg.config_load().guid
    product_family = "Isilon"
    user_agent = "isi_gather_info"
    custom_headers = (("SerialNumber", serial_number),
                      ("GUID", guid),
                      ("ProductFamily", product_family),
                      )
    program = "/usr/local/bin/curl"
    arguments = []
    arguments.append(program)
    arguments.append("-k")
    arguments.append("-x")
    arguments.append("%s:%s" % (proxy, proxyport))
    arguments.append("--verbose")
    arguments.append("--upload-file")
    arguments.append(filepath)
    for key, value in custom_headers:
        arguments.append("--header")
        arguments.append("%s: %s" % (key, value))
    arguments.append("--user-agent")
    arguments.append(user_agent)
    arguments.append("--progress-bar")
    arguments.append("https://%s%s/%s" %
            (host, uri, os.path.basename(filepath)))

    msg = "Sending using " + " ".join(arguments)
    print msg
    log.write(msg)
    return subprocess.call(arguments)


###<Connectivity>###
class MyFTP(ftplib.FTP):
    """extends ftplib.FTP with a storbinary() that has a callback function.
        with a ProgressBar implementation that prints to stdout.
        current version of Python (2.3) doesn't have callback implemented.
    """
    def __init__(self, host='', user='', passwd='', path='', port='', \
                 proxyhost='', proxyport=''):
        global log
        self.log = log
        self.startTime  = 0
        self.fileSize   = 0
        self.uploaded   = 0
        self.enableLog  = True
        self.updateLogIntVal = 3   #interval to log to file
        self.updateTtyIntVal = 1   #interval to update progress
        self.lastLogTime = 0
        self.lastTtyTime = 0
        self.pbar = ''             #ProgressBar object

        ftplib.FTP.__init__(self)
        try:
            if proxyhost:
		print "connecting via proxy: %s " % proxyhost
                # to ftp to a proxy, we connect to the proxy, as uerser
                if(proxyport):
                    self.connect(proxyhost, proxyport)
                else:
                    self.connect(proxyhost)
            else:
                self.connect(host, port)
        except Exception, e:
            err_msg = "FTP Error: Could not connect to %s on port %s" % \
                      ((proxyhost or host), (proxyport or port))
            print e
            print err_msg
            self.log.write(err_msg)
            raise
        try:
            if proxyhost:
                #we login to the proxy with user@non_proxy_host
                #XXX this does not work with other proxy types: use urllib2
                self.login('%s@%s:%s' % (user, host, port), passwd)
            else:
                self.login(user, passwd)
        except Exception, e:
            print e
            err_msg = "FTP Error: Could not authenticate to %s with user %s " % \
                      (host, user)
            print err_msg
            self.log.write(err_msg)
            raise
        try:
            self.cwd(path)
        except Exception, e:
            print e

            print "unable to change FTP dir to %s attempting uploading to %s " \
                  % (path, self.pwd())



    def storbinary(self, cmd, fp, blocksize=8192):
        """Store a file in binary mode with callback function """

        self.voidcmd('TYPE I')
        conn = self.transfercmd(cmd)
        self._prepTransfer(fp)

        while 1:
            buf = fp.read(blocksize)
            if not buf: break
            conn.sendall(buf)
            self.cbFunc(len(buf))
        conn.close()
        return self.voidresp()

    def _prepTransfer(self, fp):
        self.fileSize = os.stat(fp.name).st_size    #set filesize
        self.startTime = time.time()                #set Starttime
        self.pbar = ProgressBar(self.fileSize)

    #callback func gets uploaded packet size
    def cbFunc(self, bufSize):
        now = time.time()
        self.uploaded += bufSize
        self.timeElapsed = now - self.startTime
        if sys.stdout.isatty() and \
        (now - self.lastTtyTime > self.updateTtyIntVal):
            self.pbar.update(self.uploaded)
            self.lastTtyTime = now

        #write to gather file
        if self.enableLog and \
        (now - self.lastLogTime > self.updateLogIntVal):
            msg = "Uploaded: %s  out of %s . Time Elapsed: %s" % \
            (format_capacity(self.uploaded), format_capacity(self.fileSize), \
             time.strftime("%H:%M:%S", time.gmtime(self.timeElapsed)))
            self.lastLogTime = now
            self.log.write(msg)
###</Connectivity>###

####<XML CONFIG>###
class MXDconfig:
    """this class allows for reading/writing to the xml config file in
    /etc/mcp/ (sys &|| override)/gather.xml"""
    def __init__(self, filename='gather.xml'):
        from isi.config import mxd
        import shutil

        self.filename = filename
        self.fname = CONFIGDIR + 'sys/' + filename
        self.deltafname = CONFIGDIR + 'override/' + filename
        self.rules = {}
        self.tmpdir = '/tmp/'    #have to use /tmp because TMP_SUPPORT_DIR is not defined yet

        try:
            self.doc = mxd.loadmxd(self.fname, self.deltafname)
            self.root = self.doc.getroot()
        except mxd.error, e:
            if e[0] == 22: #override/gather.xml is corrupt
                #remove old /override config and rewrite new one
                print "Error. Corrupt config file "\
                "(/etc/mcp/override/gather.xml)"
                print "Resetting config file."
                tempConfigFilePath = self.tmpdir + self.filename
                try:
                    self.doc = mxd.loadmxd(self.fname)
                    self.root = self.doc.getroot()
                    self.doc.save(tempConfigFilePath)
                    shutil.move(tempConfigFilePath, self.deltafname)
                except mxd.error, e:
                    print "Reset Failed: Using given "\
                    "command line options"
                    pass

    def getconfig(self):
        """parses xml files and returns a dict"""
        try:
            if self.root != None:
                #gathers the 'settings' node of the xml file and makes a \
                #dictionary
                for each in self.root.getchildren():
                    if len(each.gettext()) > 0:
                        self.rules[each.getattributes()['name']] = \
                                each.gettext()
            return self.rules
        except Exception, e:
            print "Error with config file. %s" % str(e)

    def setconfig(self, rules):
        """takes a dict of rules and saves them"""
        if rules != {}:
            for key in rules.keys():
                for each in self.root.getchildren():
                    if each.getattributes()['name'] == key:
                        each.settext(str(rules.get(key)))
            tempConfigFilePath = self.tmpdir + self.filename
            self.doc.save(tempConfigFilePath)                       #save the file in tmp directory
            shutil.move(tempConfigFilePath, self.deltafname)        #move it to config directory.

def saveConfigOptions():
    """compiles the settings and saves them to the xml config files at
     /etc/mcp/(sys &| override)/gather.xml"""
    global options
    rules = {}  #preps a dict of options to write to config xml
    #these are the options from the cmd line to be saved to xml.
    #the items correspond to keys in "OPTIONS" dict
    optionsToSave = ['incremental', 'ftp_host', 'ftp_port', 'ftp_path', 'ftp_user',
                     'ftp_proxy_host', 'ftp_proxy_port', 'ftp_pass', 'http_host', 'http_path',
                     'email_addresses', 'email_subject',
                     'email_body', 'ftp_enable', 'http_enable',
                     'upload_package', 'email_enable']
    for each in optionsToSave:
        try:
            if options.__dict__[each]:
                rules.update({each : options.__dict__[each]})
        except KeyError:
            pass
        except Exception, e:     #allow for clean failures
            print "Error saving config: %s" % str(e)

    #take care of non string config settings
    rules.update({'incremental':    options.__dict__['incremental']})
    rules.update({'upload_package': options.__dict__['upload_package']})
    rules.update({'email_enable': options.__dict__['email_enable']})
    rules.update({'http_enable': options.__dict__['http_enable']})
    rules.update({'ftp_enable': options.__dict__['ftp_enable']})

    config = MXDconfig()
    config.setconfig(rules)
    return


def loadConfigOptions():
    """loads the xml config options into the OPTIONS object"""
    global options

    try:
        config = MXDconfig()
        rules = config.getconfig()
        options.__dict__.update(rules)
    except Exception, e:
        print "Error loading config options from file. Using cmdline options"

    # take care of non-string settings. Incremental is a bool
    # It is possible that these values accidentally become None.
    # Then, use default value.
    try:
        options.__dict__.update( {'incremental'   : \
                                  (rules['incremental'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'incremental': False} )
    try:
        options.__dict__.update( {'upload_package': \
                                  (rules['upload_package'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'upload_package': True} )
    try:
        options.__dict__.update( {'email_enable': \
                                  (rules['email_enable'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'email_enable': False} )
    try:
        options.__dict__.update( {'ftp_enable': \
                                  (rules['ftp_enable'] == 'True')} )
    except KeyError:
        options.__dict__.update( {'ftp_enable': True} )
    try:
        options.__dict__.update( {'http_enable': \
                                  (rules['http_enable'] == 'True')} )
    except KeyError:
            options.__dict__.update( {'http_enable': True} )

    return

####</XML CONFIG>###

######<DEBUG/ PROGRESS INDICATORS>##########
class Dots(threading.Thread):
    """spawns a thread to prints dots to tty to indicate process activity"""
    def __init__(self, name="dots"):
        self._stopevent = threading.Event()
        self.sleeptime = 0.2
        threading.Thread.__init__(self,name=name)

    def run(self):
        while not self._stopevent.isSet() and sys.stdout.isatty():
            sys.stdout.write('.')
            sys.stdout.flush()
            time.sleep(self.sleeptime)
        if not sys.stdout.isatty():    #no need to exist anymore
            self._stopevent.set()

    def join(self, timeout=None):
        print "" #newline
        if (self.isAlive()):
            self._stopevent.set()
            threading.Thread.join(self, timeout)

class ProgressBar(object):
    """Creates a Progress indicator in a bar fashion on the tty terminal window.
        progress is updated as update is called (usually after "blocksize" of
        data is transferred)
    """
    def __init__(self, total):
        self.done               = 0
        self.ETA                = 0
        self.timeElapsed        = 0
        self.percentComplete    = 0
        self.startTime          = time.time()
        self.total              = total
        if sys.stdout.isatty():
            try:                        #import libs for determining tty width
                from fcntl import ioctl
                import termios
                #from array import array
            except ImportError:
                print "import error in fnctl and termios. screen width cannot " \
                 "be determined"
                pass

            try:
                self.screenWidth = array('h', \
                                         ioctl(sys.stdout,termios.TIOCGWINSZ,\
                                               '\0'*8))[:2][1]
            except:
                self.screenWidth = 80

    def update(self, done):
        """calculates and updates the progress.
        """
        self.done = done
        self.percentComplete = float(self.done)/float(self.total)*100
        self.timeElapsed = time.time() - self.startTime
        try:
            self.ETA = (100.0/self.percentComplete - 1)*self.timeElapsed
        except ZeroDivisionError, e:        #so it doesn't freakout if ETA
            self.ETA = 0
        sys.stdout.write( self.printer() )
        sys.stdout.flush()
        if self.percentComplete == 100:
            print

    def formatTime(self, timeX):
        """helper time function.
        Caveat: results of time functions vary between FreeBSD and Linux
        """
        return time.strftime("%H:%M:%S",time.gmtime(timeX))

    def printer(self):
        """creates the text string to be displayed. ex:
        90.00% [#################################---] ETA: 00:00:01 Elapsed: 00:00:03 2880/3200 KB
        """
        lhs = "%.2f%% [" % self.percentComplete
        rhs = "]"
        rhs += " ETA: " + self.formatTime(self.ETA)
        rhs += " Elapsed: " + self.formatTime(self.timeElapsed)
        rhs += " %s/%s " %( format_capacity(self.done),
                                format_capacity(self.total))
        #lspace = space to left justify
        lspace = self.screenWidth - len(str(lhs)) - len(str(rhs))
        bar = "#"*int((self.percentComplete/100.0)*lspace)
        bar = bar.ljust(lspace).replace(" ","-")
        out = lhs + bar + rhs + '\r'
        return out
##########</DEBUG/ PROGRESS INDICATORS>##########

class MyLock(object):
    """class to handle .lock files
        """
    def __init__(self, path = GATHER_STATUS_LOCK, verbose = False):
        self.lockFD = ''
        self.lockfile = path
        self.pid = os.getpid()
        self.verbose = verbose

    def isPidRunning(self, pid):
        """checks if a given isi_gather_info is an active process"""
        cmd = """ps -o command= -p %d """
        try:
            (exit, output) = procs.get_cmd_output(cmd % pid)
            if exit:
                return False
            if output and len(output):
                #if we get an output, it'll be just one element -> pid
                return 'isi_gather_info' in output[0]
            return False
        except:
            return False

    def _readlock(self):
        """reads the contents of a lock file. ie pid of the owner"""
        try:
            self.lockFD=os.open(self.lockfile, os.O_RDONLY)
            pid = os.read(self.lockFD, 100)
            os.close(self.lockFD)
            if pid != '':
                pid = int(pid)
            else:
                pid = 10**8
            return pid
        except:
            return False

    def _lock(self):
        """helper to lock()"""
        try:
            self.lockFD = os.open(self.lockfile, \
                    os.O_EXCL | os.O_CREAT | os.O_RDWR, 0400)
            os.write(self.lockFD, str(self.pid))
            os.close(self.lockFD)
            return True
        except OSError,e:
            if e.errno == 17:   #file exists!
                return False
            return False
        except:                 #any others
            return False

    def ownlock(self):
        """do i own lock"""
        try:
            lock = self._readlock()
            if lock:
                return self.pid == lock
            return False
        except:
            return False

    def lock(self):
        """locks. writes own pid to lock file"""

        #is gather-status unlocked? try locking
        if not os.path.exists(self.lockfile):
            return self._lock()
        else:
            pid = self._readlock()
            #is gather-status.lock owner running? ie lock expired?
            if not self.isPidRunning(pid):
                print "Unlocking gather-status"
                self.unlock()
                return self._lock()
            else:
                print "%s is currently locked. Status Updates will not be written to file." \
                % (self.lockfile)
        return False

    def unlock(self):
        """ unlocks lockfile """
        if os.path.exists(self.lockfile):
            print "Gather-status unlocked"
            try:
                os.remove(self.lockfile)
            except Exception, e:
                print "Error unlocking lockfile"
                return False
            return True
        else:
            print ("Gather-status lock does not exit")
            return False

class Logger(threading.Thread):
    """runs logging to file as a thread. timestamps the log file every
        self.updateInterval seconds. also allows async writes, as used
        to update program status.
    """
    def __init__(self, name="Logger", filename = GATHER_STATUS_PATH):
        self._stopevent = threading.Event()
        self.updateInterval = 3
        self.lastLogTime = 0
        self.fileFD = ''
        self.filename = filename
        self.disable = False
        self.flock = MyLock()

        self.flock.owner = self.flock.lock()
        if self.flock.owner:
            try:
                self.fileFD = open(self.filename, 'w')
            except IOError, e:
                print "Could not open gather-status file. No igi status will "\
                "be stored."
                self.disable = True
        else:
            self.disable = True
        threading.Thread.__init__(self, name=name)
        self.start()

    def run(self):
        """timestamps the file as long as the thread is active"""
        while (not self._stopevent.isSet()) and (not self.disable):
            self.write()
            time.sleep(self.updateInterval)

    def write(self, msg="", priority=False):
        """priority write. update with msg"""
        if (not self._stopevent.isSet()) and (not self.disable):
            try:
                self.lastLogTime = time.time()
                timestamp = "[%s] " % time.strftime("%Y-%m-%d %H:%M:%S")
                self.fileFD.writelines(timestamp + msg + '\n')
                self.fileFD.flush()
            except IOError, e:
                #this is to escape any errors generated by a closed terminal
                self.join(None)
                pass

    def join(self, timeout=None):
        self._stopevent.set()
        if self.flock.owner:
            try:
                self.fileFD.close()
                self.flock.unlock()
                threading.Thread.join(self, timeout)
            except Exception,e:
                print "Error unlocking gather log"
                pass

def extendCLOptionsDefault(option, defaultParam):
    """extends the default optional parameter of a cmdline option. mainly
       useful for using with getopts.
    ex: isi_gather_info --incremental becomes isi_gather_info --incremental on
    """
    try:
        if (option in sys.argv[1:]):
            #check to see its not the last one
            if sys.argv.index(option) != len(sys.argv)-1:
                #check to see if following text is option
                if sys.argv[sys.argv.index(option)+1].startswith('-'):
                    sys.argv.insert(
                            sys.argv.index(option)+1,
                            defaultParam)
            else:
                sys.argv.insert(
                        sys.argv.index(option)+1,
                        defaultParam)
    except Exception,e:
        print "Error in parsing non argument option. %s" % str(e)

def get_options():
    global cluster, options, log, re_upload

    ##parse "--incremental" option as "--incremental on"
    extendCLOptionsDefault('--incremental', 'on')

    try:
        opts, args = getopt.getopt(sys.argv[1:],
            'hvlu:p:f:i:x:n:1:s:S:a:A:t:T:IL',
            ['help', 'version', 'list', 'user=', 'password=',
             'include=', 'nodes=', 'tarfile=', 'temp=', 'ifs', 'incremental=',
             'nologs', 'debug', 'verbose', 'varlog_all', 'varlog_recent',
             'upload', 'noupload', 'ftp', 'noftp', 'ftp-user=', 'ftp-pass=',
             'ftp-host=', 'ftp-path=', 'ftp-mode=', 'skip-node-check', 'save',
             'save-only', 'no-config', 'http', 'nohttp', 'http-host=',
             'http-path=', 'verify-upload', 'http-proxy=', 'http-proxy-port=',
             'ftp-port=', 'ftp-proxy=', 'ftp-proxy-port=',
             're-upload=', 'email', 'noemail', 'email-addresses=',
             'email-from=', 'email-body=', 'email-subject=', 'skip-size-check',
             'group=', 'incremental=', 'noconfig', 'tardir=',
             'symlinkdir=', 'local-only', 'esrs', 'no-dumps', 'clean-cores', 'clean-dumps',
             'clean-all', 'no-cores', 'dumps', 'cores'
     ])

        # load saved options unless --noconfig is specified
        if '--noconfig' not in [i[0] for i in opts]:
            loadConfigOptions()

        for o, a in opts:
            if o in ['-h', '--help']:
                usage(0)
            elif o in ['-v', '--version']:
                version()
            elif o in ['-l', '--list']:
                listUtils()
                listGroups()
                sys.exit(0)
            elif o in ['--noconfig']:
                pass     # nothing to do here; handled above
            elif o in ['-u', '--user']:
                cluster.user = a
            elif o in ['-p', '--password']:
                cluster.password = a
            elif o in ['-f']:
                options.extra_file_list.extend(a.split(','))
            elif o in ['-i', '--include']:
                options.includes.extend(a.split(','))
            elif o in ['-x']:
                for each in a.split(','):
                    try:
                        EXEMPT.remove(each)
                    except ValueError, e:
                        error("Utility %s does not exist to be excluded." % \
                              (each))
            elif o in ['-n', '--nodes']:
                options.isi_nodes_args += [o, a]
            elif o in ['--local-only']:
                options.local_only = True
            elif o in ['--skip-node-check']:
                options.skip_node_check = True
            elif o in ['-1']:
                if not is_compliance():
                    options.gather_expr_local.extend(a.split(','))
                else:
                    error('gather_exprs are not supported in ' + \
                            'compliance mode.')
            elif o in ['-s']:
                if not is_compliance():
                    options.gather_script.extend(a.split(','))
                else:
                    error('gather scripts are not supported in ' + \
                            'compliance mode.')
            elif o in ['-S']:
                if not is_compliance():
                    options.gather_expr.extend(a.split(','))
                else:
                    error('gather_exprs are not supported in ' + \
                            'compliance mode.')
            elif o in ['-a']:
                if not is_compliance():
                    options.analysis_script.extend(a.split(','))
                else:
                    error('analysis scripts are not supported in ' + \
                            'compliance mode.')
                options.analysis_script.extend(a.split(','))
            elif o in ['-A']:
                if not is_compliance():
                    options.analysis_expr.extend(a.split(','))
                else:
                    error('analysis_exprs are not supported in ' + \
                            'compliance mode.')
                options.analysis_expr.extend(a.split(','))
            elif o in ['-t', '--tarfile']:
                options.tarfile = a
            elif o in ['--tardir']:
                # this is a lot like -T, except -T still creates a 'pkg'
                # directory underneath the specified directory, which is
                # not always desirable.  I'd replace -T but I have no
                # idea if its being used.
                if not a.startswith('/'):
                    print "Tar directory %s invalid; it must begin with /"
                    sys.exit(1)

                options.tardir = a
            elif o in ['--symlinkdir']:
                if not a.startswith('/'):
                    print "Symlink dir %s invalid; it must begin with /"
                    sys.exit(1)

                options.symlinkdir = a
            elif o in ['-T', '--temp']:
                options.tempdir = a.lstrip()
            elif o in ['-I', '--ifs']:
                options.use_ifs = True
            elif o in ['-L', '--local-storage']:
                options.use_ifs = False
            elif (o in ['--incremental']):
                if a == 'off':
                    options.incremental = False
                else:
                    options.incremental = True
            elif o in ['--nologs']:
                options.no_exempt_logs = True
            elif o in ['--debug']:
                options.debug = True
            elif o in ['--verbose']:
                options.verbose = True
            elif o in ['--varlog_all']:
                # This is the default
                EXEMPT[0] = 'varlog_all'
            elif o in ['--varlog_recent']:
                EXEMPT[0] = 'varlog_recent'
            elif o in ['--noupload']:
                options.upload_package = False
            elif o in ['--upload']:
                options.upload_package = True
            elif o in ['--noftp']:
                options.ftp_enable = False
            elif o in ['--ftp']:
                options.ftp_enable = True
            elif o in ['--ftp-user']:
                options.ftp_user = a
            elif o in ['--ftp-pass']:
                options.ftp_pass = a
            elif o in ['--ftp-host']:
                options.ftp_host = a
            elif o in ['--ftp-path']:
                options.ftp_path = a
            elif o in ['--ftp-port']:
                options.ftp_port = a
            elif o in ['--ftp-mode']:
                if a == "active":
                   options.try_active_ftp = True
                   options.try_passive_ftp = False
                elif a == "passive":
                   options.try_active_ftp = False
                   options.try_passive_ftp = True
                elif a == "both":
                   options.try_active_ftp = True
                   options.try_passive_ftp = True
                else:
                   sys.stderr.write("Valid options for --ftp-mode are: "\
                                    "both, active and passive")
                   usage(1)
            elif o in ['--ftp-proxy']:
                options.ftp_proxy_host = a
            elif o in ['--noemail']:
                options.email_enable = False
            elif o in ['--email']:
                options.email_enable = True
            elif o in ['--email-addresses']:
                options.email_addresses = a
            elif o in ['--email-from']:
                options.email_from = a
            elif o in ['--email-subject']:
                options.email_subject = a
            elif o in ['--email-body']:
                options.email_body = a
            elif o in ['--skip-size-check']:
                options.skip_size_check = True
            elif o in ['--ftp-proxy-port']:
                options.ftp_proxy_port = a
            elif o in ['--nohttp']:
                options.http_enable = False
            elif o in ['--http']:
                options.http_enable = True
            elif o in ['--http-host']:
                options.http_host = a
            elif o in ['--http-path']:
                options.http_path = a
            elif o in ['--http-proxy']:
                options.http_proxy = a
            elif o in ['--http-proxy-port']:
                options.http_proxy_port = a
            elif o in ['--esrs']:
                options.esrs = True
            elif o in ['--cores']:
                options.cores = True
            elif o in ['--dumps']:
                options.dumps = True
            elif o in ['--clean-cores']:
                options.clean_cores = True
            elif o in ['--clean-dumps']:
                options.clean_dumps = True
            elif o in ['--clean-all']:
                options.clean_cores = True
                options.clean_dumps = True
            elif o in ['--no-dumps']:
                try:
                    EXEMPT.remove('isi_hangdump')
                except ValueError, e:
                    error("isi_hangdump does not exist to be excluded.")
            elif o in ['--no-cores']:
                try:
                    EXEMPT.remove('cores')
                except ValueError, e:
                    error("cores does not exist to be excluded.")
            elif o in ['--verify-upload']:
                options.verify_upload = True
            elif o in ['--save']:
                saveConfigOptions()
            elif o in ['--save-only']:
                options.save_only = True
            elif o in ['--re-upload']:
                options.re_upload = True
                re_upload = a
            elif o in ['--group']:
                try:
                    options.group_utils.extend(GROUPS[a])
                    options.group.append(a)
                except KeyError, e:
                    print "Invalid group specified: %s" % a
                    listGroups()
                    sys.exit(1)
            else:
                print "Unknown argument: %s" % o
                usage(1)

    except getopt.GetoptError, e:
        error(str(e.msg))

    if is_compliance():
        cluster.user = 'compadmin'

def usage(exitCode):
    sys.stderr.write('''
Usage: %s [OPTION]...

Options:
  -h                  Print this message and exit.
  -v                  Print version info and exit.
  -u USER             Login as USER instead of default root.
  -p PASSWORD         Use PASSWORD.
  -i                  Include only the listed utility.  See also the
                      -l option for a list of utilities to include.
                      The special value "all" may be used to include
                      every known utility.
  --incremental       Gather only logs changed since last log upload
  -l                  List utilities and groups which can be included
                      (see -i and --group)
  -f FILENAME         Gather FILENAME from each node.
  -n NODES            Gather information only from the given nodes.
                      Must be a list or range of LNNs (what you see
                      in "isi stat" output). For example: '1,4-10,12,14'.
                      If no nodes are explicitly listed, the whole array
                      is used.  Note that nodes are automatically
                      excluded if they are down.
  --local-only        Do not gather any information from other nodes.
  --skip-node-check   Skip check for node availability.
  -s gather_script    *Run gather_script on every node.
  -S gather_expr      *Run gather_expr on every node.
  -1 gather_expr      *Run gather_expr on the local node.
  -a analysis_script  *Run analysis_script on results.
  -A analysis_expr    *Run analysis_expr on every node.
  -t TARFILE          Save all results to TARFILE instead of default tar file.
  -x exclude_tool     Excludes the specified tool from being gathered from each node.
  -I                  Save results to IFS (default)
  -L                  Save all results to local storage: %s
  -T TEMPDIR          Save all results to TEMPDIR instead of default dir.
                      (overrides -L and -I)
  --tardir <dir>      Place the final package directly into <dir>.
  --symlinkdir <dir>  Create a symlink to the final package in <dir>.
  --varlog_recent     Gather all logs in /var/log, but not the compressed and
                      rotated old logs (Default is for all logs).
  --varlog_all        Gather all logs in /var/log, including compressed and
                      rotated old logs (Default).
  --nologs            Do not gather normal required minimal logs.
  --group <name>      Add a specific group of utilities to the tarball.
  --clean-cores       Delete cores from /var/crash after successful compression of the package.
  --clean-dumps       Delete dumps from /var/crash after successful compression of the package.
  --clean-all         Delete cores and dumps from /var/crash after successful compression of the package.
  --no-dumps          Don't get hangdumps.
  --dumps             Add cores to the package.
  --no-cores          Don't get cores.
  --cores             Add dumps to the package.

Configuration:
  --noconfig         Use built-in defaults and bypass the configuration file.
  --save-only        Save the CLI specified configuration to file and exit.
  --save             Save the CLI specified configuration to file and run.

Upload Options:
  --upload            Upload logs to isilon automatically. (Default)
  --noupload          Do not upload logs to isilon automatically.
  --re-upload FILE    Re-uploads the specified FILE.
  --verify-upload     Creates a tar file and uploads to test connectivity.

  HTTP Upload Options:
    --http                 Attempt HTTP upload. (Default)
    --nohttp               Do not attempt HTTP upload.
    --http-host HOST       Specifies alternate HTTP site for upload.
    --http-path DIR        Specifies alternate HTTP upload directory.
    --http-proxy HOST      Specifies Proxy server to use.
    --http-proxy-port PORT Specifies Proxy port to use.

  FTP Upload Options:
    --ftp                 Attempt FTP upload. (Default)
    --noftp               Do not attempt FTP upload.
    --ftp-user USER       Specify alternate user for FTP.  (Default: anonymous)
    --ftp-pass PASS       Specify alternate password for FTP.
    --ftp-host HOST       Specifies alternate FTP site for upload.
    --ftp-path DIR        Specifies alternate FTP upload directory.
    --ftp-port PORT       Specifies alternate FTP port for upload.
    --ftp-proxy HOST      Specifies Proxy server to use.
    --ftp-proxy-port PORT Specifies Proxy port to use.
    --ftp-mode MODE       Mode of FTP file transfer (default: attempt both)
                          valid: both, active, passive

  ESRS Upload Options:
    --esrs                Attempt ESRS upload.

  SMTP Upload Options:
    --email             Attempt SMTP upload. (If set, SMTP is tried first.)
    --noemail           Do not attempt SMTP upload. (Default)
    --email-addresses   Specify email addresses. (separated by comma)
    --email-from        Specify sender email address.
    --email-subject     Specify alternative email subject.
    --email-body        Specify alternative email text shown on head of body.
    --skip-size-check   Do no check size of gathered file.

  Multiple instances of -i, -f, -s, -S, -1 are allowed.
  gather_expr, analysis_expr can be quoted.
  Default temporary directory is %s
  (change with -L or -T)

  * Not available on compliance mode.

''' % (PROGNAME, VAR_SUPPORT_DIR, IFS_SUPPORT_DIR))
    sys.exit(exitCode)

def version():
    print '%s version: %s' % (PROGNAME, REVISION)
    sys.exit(0)

if __name__ == '__main__':
    main()
