#!/usr/bin/env python

# Copyright C 2002-2008 by Isilon Systems, Inc. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
#
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
#
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems.
#
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.
#
# $Id$

import errno
import getopt
import os
import shutil
import signal
import sys
import time
from collections import defaultdict

import isi.config.ifs as ifsc
import isi.config.array as array
import isi.net.flexnet as flx
import isi.misc
from isi.app.lib import procs as procs

NETLOGDIR = '/ifs/netlog/'
PIDFILE = '/ifs/.ifsvar/run/isi_netlogger.pid'
NODEFILE = '/ifs/.ifsvar/run/isi_netlogger.node'
LOGFILE = '/ifs/.ifsvar/run/isi_netlogger.log'
LCKFILE = '/ifs/.ifsvar/run/isi_netlogger.lck'


def isys(cmd):
    try:
        (error, output) = procs.get_cmd_output(cmd)
        if not error:
            return map(lambda x: x.rstrip(), output)
        else:
            print error, output
    except Exception as e:
        print "ERROR: ", e
    return None


def cleanup(logdir, keep):
    global node_ifaces_info
    if keep > 0:
        node_name = os.path.basename(logdir)
        if node_ifaces_info[node_name]:
            for iface in node_ifaces_info[node_name]:
                files = isys("ls -tr %s/*%s.%s.*" % (logdir, node_name, iface))
                if not files:
                    continue
                elif len(files) < 1:
                    continue

                count = len(files)
                for f in files:
                    if count <= keep:
                        break

                    os.unlink(f)
                    count = count - 1
        else:  ### Interface not specified
            files = isys("ls -tr %s/*" % logdir)
            if not files:
                return
            elif len(files) < 1:
                return

            count = len(files)
            for f in files:
                if count <= keep:
                    break

                os.unlink(f)
                count = count - 1


def get_id():
    return ifsc.get()['current node id']


def get_default_iface():
    global myid
    try:
        myid = get_id()
        lnis = flx.lni_query(myid, flx.LNI_ANY_IDX, flx.IFFLAG_EXTERNAL)
        return lnis[0].nic.name
    except Exception as e:
        print "Error: ", str(e)
        return None


### This function actually does the following #########
### 1. Verify that the interface is usable and active on the node
### 2. Call nic_load from flexnet module
def get_nicname_from_node(node_name, iface):
    global myid
    global active_lnis_nics
    try:
        NodeId = NodeNameToNodeId(node_name)

        #### Check if the interface is usable and active and up
        iface_active = False
        for lni, nic in active_lnis_nics[NodeId]:
            if nic == iface:
                iface_active = True
                break
        if not iface_active:
            verbose("On Node %s, Interface %s is either unusable or down or inactive" % (node_name, iface))
            sys.exit(1)

        if NodeId == myid:
            try:
                nic = flx.nic_load(iface)
                nic_name = nic.name
            except Exception as e:
                print "Error: ", str(e)
                sys.exit(1)
        else:
            cmd = "isi_rdo %s isi_nic_load -i %s" % (node_name, iface)
            exit, output = procs.get_cmd_output(cmd)
            if exit:
                print "Error invoking isi_nic_load on node %s: %s" % (node_name, str(output))
                sys.exit(1)
            else:
                if output[0] == 'None':
                    verbose("On node %s, Unable to load interface '%s'" % (node_name, iface))
                    sys.exit(1)
                else:
                    nic_name = output[0]
        return nic_name
    except Exception as e:
        print "Error: ", str(e)
        return ''


def get_hostname():
    ret = isys("/bin/hostname")[0]
    return ret


###################################################
### This function assumes the following:
### 1. nodes_info has already been populated
### 2. Bufsize specified with -b option is appropriate for
###    further running of the system. No check is done on
###    the value as such
def process_bpf_bufsize(o, a):
    """Process and set bpf_bufsize"""

    ### The following is same for all nodes

    ### Set this to 0 to check later if this has been set
    new_bpf_bufsize = 0

    ### Now parse -b option argument
    if a[-1] == 'k' or a[-1] == 'K':
        new_bpf_bufsize = int(a[0:-1]) * 1024
    elif a[-1] == "m" or a[-1] == "M":
        new_bpf_bufsize = int(a[0:-1]) * 1024 * 1024
    else:
        new_bpf_bufsize = int(a)

    # Don't let it fall below FreeBSD's default bpf_bufsize
    if new_bpf_bufsize < 4096:
        new_bpf_bufsize = 4096

    ### The following depends on original maxbufsize
    for i in range(len(nodes_info)):
        node_name, node_address = nodes_info[i].split()

        ORIG_BPF_BUFSIZE = int(get_sysctl(node_name, "net.bpf.bufsize"))
        ORIG_BPF_MAXBUFSIZE = int(get_sysctl(node_name, "net.bpf.maxbufsize"))
        # Make bpf_maxbufsize at least twice as big as bpf_bufsize
        if new_bpf_bufsize > ORIG_BPF_MAXBUFSIZE / 2:
            new_bpf_maxbufsize = new_bpf_bufsize * 2
        ####### Else set this to 0 to indicate that this has not been changed
        else:
            new_bpf_maxbufsize = 0

        node_bpf_bufsize[node_name] = {'ORIG': ORIG_BPF_BUFSIZE, 'ORIG_MAX': ORIG_BPF_MAXBUFSIZE,
                                       'NEW': new_bpf_bufsize, 'NEW_MAX': new_bpf_maxbufsize}

        if new_bpf_bufsize:
            if not set_sysctl(node_name, "net.bpf.bufsize", str(new_bpf_bufsize)):
                verbose("Failed to set bpf buffer size to", new_bpf_bufsize)
                sys.exit(1)

        if new_bpf_maxbufsize:
            if not set_sysctl(node_name, "net.bpf.maxbufsize", str(new_bpf_maxbufsize)):
                set_sysctl(node_name, "net.bpf.bufsize", ORIG_BPF_BUFSIZE)
                verbose("Failed to set bpf max buffer size to", new_bpf_maxbufsize)
                sys.exit(1)


def restore_bpf_bufsize(signum=0, frame=0):
    """Restore bpf_bufsize values on all nodes"""
    for i in range(len(nodes_info)):
        node_name, node_address = nodes_info[i].split()

        set_sysctl(node_name, "net.bpf.bufsize", str(node_bpf_bufsize[node_name]['ORIG']))
        set_sysctl(node_name, "net.bpf.maxbufsize", str(node_bpf_bufsize[node_name]['ORIG_MAX']))


def plural(count):
    try:
        if count == 1:
            return ""
        else:
            return "s"
    except Exception as e:
        print "Error: ", str(e)
        return ""


def get_sysctl(node_name, s):
    global myid
    myid = get_id()

    NodeId = NodeNameToNodeId(node_name)
    if NodeId == myid:
        toks = isys("sysctl " + s + " 2>/dev/null").pop().split()
    else:
        cmd = "isi_rdo %s sysctl %s 2>/dev/null" % (node_name, s)
        exit, output = procs.get_cmd_output(cmd)
        if exit:
            print "Error invoking sysctl on node %s: %s" % (node_name, str(output))
            sys.exit(1)
        else:
            toks = output.pop().split()

    if len(toks) == 2:
        return toks[1]
    else:
        return ""


def set_sysctl(node_name, s, val):
    global myid
    myid = get_id()

    NodeId = NodeNameToNodeId(node_name)
    if NodeId == myid:
        toks = isys("sysctl " + s + "=" + val + " 2>/dev/null").pop().split()
    else:
        cmd = "isi_rdo %s sysctl %s=%s 2>/dev/null" % (node_name, s, val)
        exit, output = procs.get_cmd_output(cmd)
        if exit:
            print "Error invoking sysctl on node %s: %s" % (node_name, str(output))
            sys.exit(1)
        else:
            toks = output.pop().split()

    if len(toks) == 4:
        return 1
    else:
        return 0


def daemonize():
    verbose("Daemonizing")
    try:
        pid = os.fork()
        if pid:
            sys.exit(0)
    except OSError as e:
        print "Fork #1 failed. Can not daemonize. %s" % str(e)
        sys.exit(1)

    os.setsid()
    os.umask(0)
    os.chdir('/')
    try:
        npid = os.fork()
        if npid:
            sys.exit(0)
    except OSError as e:
        print "Fork #2 failed. Can not daemonize. %s" % str(e)
        sys.exit(1)

    f = open(LOGFILE, 'w')
    f.write(isys("date +\"%Y-%m-%d_%H.%M.%S\"")[0])
    si = file('/dev/null', 'r')
    os.dup2(si.fileno(), sys.stdin.fileno())
    os.dup2(f.fileno(), sys.stdout.fileno())
    os.dup2(f.fileno(), sys.stderr.fileno())
    print "Running isi_netlogger as a daemon"


def syntax(logdir):
    print """\
isi_netlogger [-c launch clustered mode (run on all nodes)]
              [-n run on specified nodes (ex: -n 1,3)]
              [-d run as daemon]
              [-q quiet mode (redirect output to logs)]
              [-k keep_count of logs (default 3, to keep all logs use 0)]
              [-t timeout (default 10) ]
              [-s snaplen (default 320) ]
              [-b bpf buffer size in bytes, KB, or MB (end with 'k' or 'm') ]
              [-i interface name[,..] | all (ex: -i em0 or -i em0,em1 or -i all)]
              [-a ARP packets included. Normally filtered out ]
              [-p print out the tcpdump command ]
              [-z do not bundle capture files (default bundling is done)]
              [-- tcpdump filtering options]

isi_netlogger captures ip traffic on the specified interface over
a period of timeout minutes, and stores keep_count log files of
them. Using the bpf buffer size option will temporarily change the
default bpf buffer size while isi_netlogger is running."""
    print "Logs for this node will be stored in %s." % logdir


def mk_pid_file():
    global isi_netlogger_pid
    try:
        isi_netlogger_pid = os.getpid()
        f = open(PIDFILE, 'w')
        f.write("%d" % (isi_netlogger_pid))
        f.close()
    except Exception as e:
        verbose("Failed to write pid to pidfile: %s" % str(e))
        # Fail? since lack of ppid => can't kill child pids
        if os.path.exists(PIDFILE):
            os.remove(PIDFILE)
        sys.exit(1)


def mk_node_file():
    global isi_netlogger_node_file
    try:
        local_node_lnn = str(array.getLocalNode().lnn())
        isi_netlogger_node_file = os.open(NODEFILE, os.O_CREAT | os.O_WRONLY | os.O_EXLOCK ,0755)
        os.write(isi_netlogger_node_file, local_node_lnn.zfill(3))
        # leave open until cleanup holding advisory lock
    except Exception, e:
        # only used by others
        pass

def quiet_mode():
    try:
        f = open(LOGFILE, 'w')
        sys.stdout = f
        sys.stderr = f
    except Exception as e:
        print sys.stdout, str(e)
    verbose("Starting " + time.ctime())  # to keep track of when this was run

def verbose(*args):
    """prints messages to console or to log file based on the boolean options.quietmode"""
    msg = "".join([str(each) for each in args])
    print msg
    sys.stdout.flush()

def parse_nodes(cliParams):
    "parses cli node info to isi_node formatted info"
    #cmd: isi_nodes -n "1,2-5,1" %{name} %{internal}

    cmd = "isi_nodes %s %s"
    specifiers = " %{name} %{internal}"
    cliParams = ''.join(cliParams)
    exit, output = procs.get_cmd_output(cmd % (cliParams, specifiers))
    if exit:
        print "Error parsing node information: %s" % str(output)
        sys.exit(1)
    else:
        return output


##############################################################
### parse_node_ifaces function assumes the following #########
### 1. nodes_info has already been populated
### 2. There is no inconsistency between nodes specified using -c or -n
###    options and those specified in -i option. Any way, the code handling
###    nodes in -i option is not being included at this time and hence has been
###    commented out below.
def parse_node_ifaces(o, a):
    "parses cli node interfaces info"
    global myid
    myid = get_id()
    #### All interfaces but no node specified in -i option
    if a.strip().lower() == 'all':
        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()
            SetAllifacesForNode(node_name)
        #### Multiple nodes specified in -i option
        #### This has been commented out for now
        #    elif '+' in a:
        #        multiple_nodes = a.split('+')
        #        for each_node in multiple_nodes:
        #            (nodeString, ifaces) = each_node.split(':')
        #            if nodeString.isdigit():
        #                node_name = LNNToNodeName(nodeString)
        #            else:
        #                node_name = nodeString
        #            for each_iface in ifaces.strip().split(','):
        #                if each_iface.lower() == 'all':
        #                    SetAllifacesForNode(node_name)
        #                    break
        #                else:
        #                    nic_name = get_nicname_from_node(node_name, each_iface)
        #                    if nic_name != '':
        #                        node_ifaces_info[node_name].append(nic_name)
        #                    else:
        #                        verbose( "Unable to load interface '%s'" % each_iface)
        #                        sys.exit(1)
        #### Single node specified in -i option
        #### This has been commented out for now
        #    elif ':' in a:
        #        (nodeString, ifaces) = a.split(':')
        #        if nodeString.isdigit():
        #            node_name = LNNToNodeName(nodeString)
        #        else:
        #            node_name = nodeString
        #        for each_iface in ifaces.strip().split(','):
        #            if each_iface.lower() == 'all':
        #                SetAllifacesForNode(node_name)
        #                break
        #            else:
        #                nic_name = get_nicname_from_node(node_name, each_iface)
        #                if nic_name != '':
        #                    node_ifaces_info[node_name].append(nic_name)
        #                else:
        #                    verbose( "Unable to load interface '%s'" % each_iface)
        #                    sys.exit(1)
        #### Multiple interfaces but no node specified in -i option
    elif ',' in a:
        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()
            for each_iface in a.split(','):
                if each_iface.lower() == 'all':
                    SetAllifacesForNode(node_name)
                    break
                else:
                    nic_name = get_nicname_from_node(node_name, each_iface)
                    if nic_name != '':
                        node_ifaces_info[node_name].append(nic_name)
                    else:
                        verbose("On node %s, Unable to load interface '%s'" % (node_name, each_iface))
                        sys.exit(1)
                    #### Single interface specified in -i option
    else:
        iface = a.strip()
        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()
            nic_name = get_nicname_from_node(node_name, iface)
            if nic_name != '':
                node_ifaces_info[node_name].append(nic_name)
            else:
                verbose("On node %s, Unable to load interface '%s'" % (node_name, iface))
                sys.exit(1)
    return


def make_dir(logdir):
    try:
        umask_old = os.umask(0)
        os.makedirs(logdir, 0775)
        os.umask(umask_old)
    except OSError as e:
        if e.errno != errno.EEXIST:
            verbose("Unknown error while creating logdir")
            print "Exception:", str(e)
            sys.exit(0)
    os.system("isi set -c off " + logdir)


def sig_handler(signum, frame):
    global nodes_info, cluster_children
    global isi_netlogger_pid
    global bundle_cap_files

    curr_pid = os.getpid()
    if curr_pid == isi_netlogger_pid:
        verbose("Process interrupted by signal %s." % signum)
        terminate_procs()

        if bundle_cap_files:
            date = isys("date +\"%Y-%m-%d_%H.%M.%S\"")[0]

            bundledFname = 'bundled/netlog-' + date + '.tar.bz2'
            if not os.path.exists(NETLOGDIR + 'bundled'):
                os.mkdir(NETLOGDIR + 'bundled', 0775)
            os.chmod(NETLOGDIR + 'bundled', 0775)

            nodeDirs = ''
            for each in nodes_info:
                (nodeName, nodeAddress) = each.split()
                nodeDirs += NETLOGDIR + nodeName + " "

            cmd = "tar jcf %s %s" % (NETLOGDIR + bundledFname, nodeDirs)
            exit, output = procs.get_cmd_output(cmd)
            if exit:
                verbose("Error bundling the Capture files located in (%s)." % NETLOGDIR)
            else:
                verbose("Bundling captures successful.")
                os.chmod(NETLOGDIR + bundledFname, 0664)
                for each in nodeDirs.split():
                    if os.path.exists(each):
                        shutil.rmtree(each)
                        verbose("Removing node capture folder: ", each)
        before_exit()
        sys.exit(0)
    else:  ## signal handler was called from child process
        ## before os.forkpty is complete or has just completed
        ## and the child has not yet done execv to isi_rdo
        sys.exit(0)


def before_exit():
    """things to do before shutting down"""
    global bpf_bufsize_specified
    if bpf_bufsize_specified:
        restore_bpf_bufsize()

    if os.path.exists(PIDFILE):
        os.remove(PIDFILE)
    try:
        if isi_netlogger_node_file is not None:
            os.close(isi_netlogger_node_file)
            os.unlink(NODEFILE)
    except Exception, e:
        pass

    if not sys.stdout.isatty():
        #quietMode was turned on. so close file to flush data
        sys.stdout.close()

def chk_lock_file():
    if not os.path.exists(LCKFILE):
        # end isi_nl reusing sighandler
        sig_handler(None, None)


def mk_lock_file():
    if not os.path.exists(LCKFILE):
        f = open(LCKFILE, 'w')
        f.write('')
        f.close()


def NodeNameToNodeId(node_name):
    cmd = "isi_nodes -n %s %s"
    specifiers = " %{devid}"
    exit, output = procs.get_cmd_output(cmd % (node_name, specifiers))
    if exit:
        print "Error getting Node ID from Node Name: %s" % str(output)
        sys.exit(1)
    else:
        return int(output[0])


def SetAllifacesForNode(node_name):
    global myid
    global active_lnis_nics

    devid = NodeNameToNodeId(node_name)
    for lni_name, nic_name in active_lnis_nics[devid]:
        nic_name_from_node = get_nicname_from_node(node_name, nic_name)
        if nic_name_from_node != '':
            node_ifaces_info[node_name].append(nic_name_from_node)
        else:
            verbose("On node %s, Unable to load interface '%s'" % (node_name, nic_name))
            sys.exit(1)


def GetAllActiveLnisNics():
    global active_lnis_nics
    global flxCfg
    if flxCfg is None:
        flxCfg = flx.config_load()
    for iface in flxCfg.ifaces:
        devid = iface.id.devid
        if iface.nic_status == iface.nic_status.enum.up:
            active_lnis_nics[devid].append((iface.id.lni_name,
                    iface.nic_name))

def pid_exists(pid):
    try:
        os.kill(pid, 0)
    except OSError as e:
        return False
    else:
        return True


# wait_procs: check for child status
#   print error message if a child exits non-cleanly
#   Return either when all children are done, or
#     when 'nsecs' seconds is up.
def wait_procs(nsecs):
    global cluster_children

    secs = 0
    children = {}
    for (p, fd, child_cmd) in cluster_children:
        children[p] = {}
        children[p]['fd'] = fd
        children[p]['cmd'] = child_cmd

    while secs < nsecs:
        time.sleep(2)
        secs = secs + 2
        try:
            ret_pid, status = os.waitpid(-1, os.WNOHANG)
        except OSError as e:
            if e.errno == 10:  ## ECHILD - no child process
                break
            else:
                #### Some problem with os.waitpid
                #### but we can continue rather than exiting
                pass
        else:
            if ret_pid != 0 and ret_pid in children:
                if os.WIFEXITED(status):
                    print "Command '%s' exited with exit status %d" % \
                          (children[ret_pid]['cmd'], os.WEXITSTATUS(status))

                elif os.WIFSIGNALED(status):
                    print "Command '%s' exited with signal %d" % \
                          os.WTERMSIG(status)
                try:
                    os.close(children[ret_pid]['fd'])
                except OSError as e:
                    pass
                del children[ret_pid]
                if len(children) == 0:
                    return
            else:
                #### i.e. ret_pid is 0 which occurs
                ### when none of the children has terminated
                pass


##############################################
### This function assumes the following:
### cluster_children (list of pid, fd tuples)
### has already been populated
def terminate_procs():
    global cluster_children
    global rem_children

    for (pid, fd, child_cmd) in cluster_children:
        ### Send KILL signal to isi_rdo pid
        if pid_exists(pid):
            ### Close pseudo terminal masterfd
            try:
                os.close(fd)
            ## this is possible if Ctrl-C was hit while this function was
            ## being executed at the end of an iteration (option -t)
            except OSError as e:
                pass
            rem_children.append(pid)
            os.kill(pid, signal.SIGTERM)

    ### Call wait for all remaining child processes together
    ### rather than waiting for a specific pid
    t1 = time.time()
    timeout = False
    while (len(rem_children) and not timeout):
        try:
            ret_pid, status = os.waitpid(-1, os.WNOHANG)
        ## this is possible if Ctrl-C was hit while this function was
        ## being executed at the end of an iteration (option -t)
        except OSError as e:
            if e.errno == 10:  ## ECHILD - no child process
                break
            else:  #### Some problem with os.waitpid
                #### but we can continue rather than exiting
                pass
        else:
            if ret_pid != 0 and ret_pid in rem_children:
                if os.WIFSIGNALED(status):  ### this is expected
                    rem_children.remove(ret_pid)
                else:  ### Unexpected but removing child pid any way
                    rem_children.remove(ret_pid)
            else:  #### i.e. ret_pid is 0 which occurs when none of the children has terminated or a child which was not in rem_children has terminated (due to Ctrl-C near os.forkpty)
                pass

        #### This timeout condition is being used to prevent isi_netlogger from
        #### forever looping for exit status of children if for some (unexpected)
        #### reason child processes do not terminate
        t2 = time.time()
        timeout = True if (t2 - t1) > 5 else False

# Globals (mainly used by sig handler)
cluster_children = []
nodes_info = ''
node_ifaces_info = defaultdict(list)  ## Node interfaces to be logged
node_bpf_bufsize = defaultdict(tuple)
rem_children = []  #### Remaining children whose exit status to wait for
isi_netlogger_pid = -1
active_lnis_nics = defaultdict(list)  ## All active interfaces on all nodes
flxCfg = None
bundle_cap_files = True


def main(argv):
    global cluster_children, nodes_info
    global node_ifaces_info
    global myid
    global logdir
    global bpf_bufsize_specified
    global bundle_cap_files

    sleep_mins = 10
    keep_count = 3
    iface = get_default_iface()
    hostname = get_hostname()
    logdir = os.path.join(NETLOGDIR, hostname)
    snaplen = 320
    bpf_bufsize_specified = False
    print_cmd = False
    arp_included = False
    daemon = False
    ifaces_specified = False

    try:
        opts, args = getopt.getopt(argv[1:], "k:i:t:s:b:pan:chqdz")
    except getopt.GetoptError, e:
        verbose("Invalid argument: ", e)
        syntax(logdir)
        return

    tcpargs = []
    try:
        for o, a in opts:
            if o == '-d':
                daemon = True
                # do not pass to tcp args
            elif o == "-k":
                keep_count = int(a)
                tcpargs.append('-W')
                tcpargs.append(a)
            elif o == "-t":
                sleep_mins = int(a)
                # do not pass to tcp args
            elif o == '-i':
                ifaces_specified = True
                iface_o = o
                iface_a = a
                # do not pass to tcp args; handled elsewhere
            elif o == "-s":
                snaplen = int(a)
                tcpargs.append('-s')
                tcpargs.append(a)
            elif o == "-h":
                syntax(logdir)
                return
            elif o == "-b":
                bpf_bufsize_specified = True
                bpf_o = o
                bpf_a = a
                # do not pass to tcp args
            elif o == "-p":
                print_cmd = True
                # do not pass to tcp args
            elif o == "-a":
                arp_included = True
                # do not pass to tcp args
            elif o == '-n':
                nodes_info = parse_nodes([o, a])
                # do not pass to tcp args
            elif o == '-c':
                nodes_info = parse_nodes('')
                # do not pass to tcp args
            elif o == '-q':
                quiet_mode()
                # options.quietmode = True
                # do not pass to tcp args
            elif o == '-z':
                bundle_cap_files = False
                # do not pass to tcp args
    except Exception as e:
        print e
        sys.exit(1)

    if sleep_mins <= 0:
        print "Timeout argument must be a positive integer"
        sys.exit(1)

    if keep_count < 0:
        print "Keep Count argument must be a non-negative integer"
        sys.exit(1)

    if keep_count:
        verbose("Netlogger keeping %d log%s of %d minute%s each..." \
                % (keep_count, plural(keep_count),
                   sleep_mins, plural(sleep_mins)))
    else:
        verbose("Netlogger creating logs of %d minute%s each..." \
                % (sleep_mins, plural(sleep_mins)))

    # make sure that we didn't inherit a bad sigmask
    isi.misc.sigprocmask(isi.misc.SIG_UNBLOCK, [signal.SIGINT, signal.SIGHUP,
            signal.SIGTSTP, signal.SIGTERM])
    signal.signal(signal.SIGINT, sig_handler)
    signal.signal(signal.SIGHUP, sig_handler)
    signal.signal(signal.SIGTSTP, sig_handler)
    signal.signal(signal.SIGTERM, sig_handler)

    if nodes_info == '':
        nodes_info = parse_nodes(['-L', ' '])

    GetAllActiveLnisNics()

    if ifaces_specified:
        parse_node_ifaces(iface_o, iface_a)

    if bpf_bufsize_specified:
        process_bpf_bufsize(bpf_o, bpf_a)

    verbose("running on nodes: ", nodes_info)

    logdirs = []
    for each in nodes_info:
        name, address = each.split()
        logdir = os.path.join(NETLOGDIR, name)
        logdirs.append(logdir)
        make_dir(logdir)

    post_args = args  # these are optional tcpdump args
    cmd = ['/usr/sbin/tcpdump -U'] + tcpargs

    if not arp_included:
        if args:
            post_args += [" and "]
        post_args += [" not arp"]

    if len(nodes_info) and os.path.exists(PIDFILE):
        print "Pid file (%s) exists. Can't run another clustered instance" % PIDFILE
        sys.exit(0)

    mk_node_file()

    if daemon:
        daemonize()

    mk_lock_file()
    mk_pid_file()

    ### This iteration number variable is being added so that printing of command
    ### can be limited to once.
    ### In future (if required), this can be used to limit isi_netlogger to specified number of iterations
    iter_no = 1
    exitval = 0

    try:
        while 1:
            chk_lock_file()
            cluster_children = []  # replace with cluster_children
            date = isys("date +\"%Y-%m-%d_%H.%M.%S\"")[0]

            for i in range(len(nodes_info)):
                node_name, node_address = nodes_info[i].split()
                pre_args = ['isi_rdo'] + [node_address]
                if node_ifaces_info[node_name]:
                    for j in node_ifaces_info[node_name]:
                        iface_args = [' -i '] + [j]
                        if print_cmd and iter_no == 1:
                            verbose("Running on node %s Command:" % (node_name), ' '.join(cmd + iface_args + post_args))
                        pst_args = ["-w", NETLOGDIR + "%s/netlog-%s.%s.%s.pcap" % (
                        node_name, node_name, j, date)] + iface_args + post_args

                        child_cmd = pre_args + cmd + pst_args
                        if print_cmd and iter_no == 1:
                            verbose("Full command is:", ' '.join(child_cmd))
                        try:
                            pid, fd = os.forkpty()
                        except OSError as e:
                            print "os.forkpty failed. %s" % str(e)
                            exitval = 1
                            raise

                        if pid == 0:
                            os.execv('/usr/bin/isi_rdo', child_cmd)
                        cluster_children.append((pid, fd, ' '.join(child_cmd)))
                else:
                    if print_cmd and iter_no == 1:
                        verbose("Running on node %s Command:" % (node_name), ' '.join(cmd + post_args))
                    pst_args = ["-w", NETLOGDIR + "%s/netlog-%s.%s.pcap" % (node_name, node_name, date)] + post_args

                    child_cmd = pre_args + cmd + pst_args
                    if print_cmd and iter_no == 1:
                        verbose("Full command is:", ' '.join(child_cmd))
                    try:
                        pid, fd = os.forkpty()
                    except OSError as e:
                        print "os.forkpty failed. %s" % str(e)
                        exitval = 1
                        raise

                    if pid == 0:
                        os.execv('/usr/bin/isi_rdo', child_cmd)
                    cluster_children.append((pid, fd, ' '.join(child_cmd)))

            wait_procs(60 * sleep_mins)
            terminate_procs()

            for each in logdirs:
                cleanup(each, keep_count)

            iter_no = iter_no + 1
    finally:
        before_exit()

    if exitval != 0:
        sys.exit(exitval)


if __name__ == "__main__":
    main(sys.argv)
