#!/usr/bin/env python

# Copyright C 2002-2008 by Isilon Systems, Inc. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
# 
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
# 
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems. 
#  
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.
#
# $Id: isi_netlogger 382385 2014-01-10 18:46:22Z csmith $

import errno
import getopt
import os
import re
import signal
import sys
import time
import shutil
from collections import defaultdict

import isi.config.ifs as ifsc
import isi.net.flexnet as flx
from isi.app.lib import procs as procs

NETLOGDIR = '/ifs/netlog/'
PIDFILE = '/ifs/.ifsvar/run/isi_netlogger.pid'
LOGFILE = '/ifs/.ifsvar/run/isi_netlogger.log'
LCKFILE = '/ifs/.ifsvar/run/isi_netlogger.lck'

def isys(cmd):
    try:
        (error, output) = procs.get_cmd_output(cmd)
        if not error:
            return map(lambda x: x.rstrip(), output)
        else:
            print error, output
    except Exception, e:
        print "ERROR: ", e

def cleanup(logdir, keep):
    global node_ifaces_info
    if keep > 0:
        node_name = os.path.basename(logdir)
        if node_ifaces_info[node_name]:
            for iface in node_ifaces_info[node_name]:
                files = isys("ls -tr %s/*%s.%s.*" %(logdir, node_name, iface))
                if not files:
                    continue 
                elif len(files) < 1:
                    continue

                count = len(files)
                for f in files:
                    if count <= keep:
                        break

                    os.unlink(f)
                    count = count - 1
        else: ### Interface not specified
            files = isys("ls -tr %s/*" % logdir)
            if not files:
                return 
            elif len(files) < 1:
                return

            count = len(files)
            for f in files:
                if count <= keep:
                    break

                os.unlink(f)
                count = count - 1

def get_id():
    return ifsc.get()['current node id']

def get_default_iface():
    global myid
    try:
        myid = get_id()
        lnis = flx.lni_query(myid, flx.LNI_ANY_IDX, flx.IFFLAG_EXTERNAL)
        return lnis[0].nic.name
    except Exception, e:
        print "Error: ", str(e)
        return None

### This function actually does the following #########
### 1. Verify that the interface is usable and active on the node
### 2. Call nic_load from flexnet module
def get_nicname_from_node(node_name, iface):
    global myid
    global active_lnis_nics
    try:
        NodeId = NodeNameToNodeId(node_name)

        #### Check if the interface is usable and active and up
        iface_active = False
        for lni, nic in active_lnis_nics[NodeId]:
            if nic == iface:
                iface_active = True
                break
        if not iface_active:
            verbose("On Node %s, Interface %s is either unusable or down or inactive" %(node_name, iface))
            sys.exit(1)

        if NodeId == myid:
            try:
                nic = flx.nic_load(iface)
                nic_name = nic.name
            except Exception, e:
                print "Error: ", str(e)
                sys.exit(1) 
        else:
            cmd = "isi_rdo %s isi_nic_load -i %s" %(node_name, iface)
            exit, output = procs.get_cmd_output(cmd)
            if exit:
                print "Error invoking isi_nic_load on node %s: %s" % (node_name,str(output))
                sys.exit(1)
            else:
                if output[0] == 'None':
                    verbose( "On node %s, Unable to load interface '%s'" %(node_name,iface))
                    sys.exit(1)
                else:
                    nic_name = output[0]
        return nic_name 
    except Exception, e:
        print "Error: ", str(e)
        return ''

def get_hostname():
    ret = isys("/bin/hostname")[0]
    return ret

###################################################
### This function assumes the following:
### 1. nodes_info has already been populated
### 2. Bufsize specified with -b option is appropriate for
###    further running of the system. No check is done on
###    the value as such
def process_bpf_bufsize(o, a):
    """Process and set bpf_bufsize"""

### The following is same for all nodes

### Set this to 0 to check later if this has been set
    new_bpf_bufsize = 0

### Now parse -b option argument
    if a[-1] == 'k' or a[-1] == 'K':
        new_bpf_bufsize = int(a[0:-1]) * 1024
    elif a[-1] == "m" or a[-1] == "M":
        new_bpf_bufsize = int(a[0:-1]) * 1024 * 1024
    else:
        new_bpf_bufsize = int(a)

    # Don't let it fall below FreeBSD's default bpf_bufsize
    if new_bpf_bufsize < 4096:
        new_bpf_bufsize = 4096

### The following depends on original maxbufsize 
    for i in range(len(nodes_info)):
        node_name, node_address = nodes_info[i].split()  

        ORIG_BPF_BUFSIZE = int(get_sysctl(node_name, "debug.bpf_bufsize"))
        ORIG_BPF_MAXBUFSIZE = int(get_sysctl(node_name, "debug.bpf_maxbufsize"))
        # Make bpf_maxbufsize at least twice as big as bpf_bufsize
        if new_bpf_bufsize > ORIG_BPF_MAXBUFSIZE / 2:
            new_bpf_maxbufsize = new_bpf_bufsize * 2
####### Else set this to 0 to indicate that this has not been changed
        else:
            new_bpf_maxbufsize = 0

        node_bpf_bufsize[node_name] = {'ORIG':ORIG_BPF_BUFSIZE, 'ORIG_MAX':ORIG_BPF_MAXBUFSIZE, 'NEW':new_bpf_bufsize, 'NEW_MAX':new_bpf_maxbufsize}

        if new_bpf_bufsize:
            if not set_sysctl(node_name, "debug.bpf_bufsize", str(new_bpf_bufsize)):
                verbose( "Failed to set bpf buffer size to", new_bpf_bufsize)
                sys.exit(1)

        if new_bpf_maxbufsize:
            if not set_sysctl(node_name, "debug.bpf_maxbufsize", str(new_bpf_maxbufsize)):
                set_sysctl(node_name, "debug.bpf_bufsize", ORIG_BPF_BUFSIZE)
                verbose( "Failed to set bpf max buffer size to", new_bpf_maxbufsize)
                sys.exit(1)

def restore_bpf_bufsize(signum = 0, frame = 0):
    """Restore bpf_bufsize values on all nodes"""
    for i in range(len(nodes_info)):
        node_name, node_address = nodes_info[i].split()  

        set_sysctl(node_name, "debug.bpf_bufsize", str(node_bpf_bufsize[node_name]['ORIG']))
        set_sysctl(node_name, "debug.bpf_maxbufsize", str(node_bpf_bufsize[node_name]['ORIG_MAX']))

def plural(count):
    try:
        if count == 1:
            return ""
        else:
            return "s"
    except Exception, e:
        print "Error: ", str(e)
        return "" 

def get_sysctl(node_name, s):
    global myid
    myid = get_id()

    NodeId = NodeNameToNodeId(node_name)
    if NodeId == myid:
        toks = isys("sysctl " + s + " 2>/dev/null").pop().split()
    else:
        cmd = "isi_rdo %s sysctl %s 2>/dev/null" %(node_name, s)
        exit, output = procs.get_cmd_output(cmd)
        if exit:
            print "Error invoking sysctl on node %s: %s" % (node_name,str(outputa))
            sys.exit(1)
        else:
            toks = output.pop().split()

    if len(toks) == 2:
        return toks[1]
    else:
        return ""

def set_sysctl(node_name, s, val):
    global myid
    myid = get_id()

    NodeId = NodeNameToNodeId(node_name)
    if NodeId == myid:
        toks = isys("sysctl " + s + "=" +  val + " 2>/dev/null").pop().split()
    else:
        cmd = "isi_rdo %s sysctl %s=%s 2>/dev/null" %(node_name, s, val)
        exit, output = procs.get_cmd_output(cmd)
        if exit:
            print "Error invoking sysctl on node %s: %s" % (node_name, str(output))
            sys.exit(1)
        else:
            toks = output.pop().split()

    if len(toks) == 4:
        return 1
    else:
        return 0

def daemonize(): 
    verbose("Daemonizing")
    try:
        pid = os.fork()
        if pid:
            sys.exit(0)
    except OSError, e:
        print "Fork #1 failed. Can not daemonize. %s" % str(e)
        sys.exit(1)

    os.setsid()
    os.umask(0)
    os.chdir('/')
    try:
        npid = os.fork()
        if npid:
            sys.exit(0)
    except OSError, e:
        print "Fork #2 failed. Can not daemonize. %s" % str(e)
        sys.exit(1)

    f = open(LOGFILE,'w')
    f.write(isys("date +\"%Y-%m-%d_%H.%M.%S\"")[0])
    si = file('/dev/null','r')
    os.dup2(si.fileno(), sys.stdin.fileno())
    os.dup2(f.fileno(), sys.stdout.fileno())
    os.dup2(f.fileno(), sys.stderr.fileno())
    print "Running isi_netlogger as a daemon"

def syntax(logdir):
    print """\
isi_netlogger [-c launch clustered mode (run on all nodes)]
              [-n run on specified nodes (ex: -n 1,3)]
              [-d run as daemon]
              [-q quiet mode (redirect output to logs)]
              [-k keep_count of logs (default 3, to keep all logs use 0)]
              [-t timeout (default 10) ]
              [-s snaplen (default 320) ]
              [-b bpf buffer size in bytes, KB, or MB (end with 'k' or 'm') ]
              [-i interface name[,..] | all (ex: -i em0 or -i em0,em1 or -i all)]
              [-a ARP packets included. Normally filtered out ]
              [-p print out the tcpdump command ]
              [-z do not bundle capture files (default bundling is done)]
              [-- tcpdump filtering options]
              
isi_netlogger captures ip traffic on the specified interface over
a period of timeout minutes, and stores keep_count log files of
them. Using the bpf buffer size option will temporarily change the
default bpf buffer size while isi_netlogger is running."""
    print "Logs for this node will be stored in %s." % logdir 

def mk_pid_file():
    global isi_netlogger_pid
    try:
        output = isys('isi_nodes -L %{internal}')
        node_address = ''
        if len(output) < 1:
            raise Exception("Could not determine node address")
        node_address = output[0]
        isi_netlogger_pid = os.getpid()
        f = open(PIDFILE,'w')
        f.write("%s|%d" % (node_address, isi_netlogger_pid))
        f.close()
    except Exception, e:
        verbose( "Failed to write pid to pidfile: %s" % str(e))
        #Fail? since lack of ppid => can't kill child pids
        sys.exit(1) 

def quiet_mode():
    try:
        f = open(LOGFILE, 'w')
        sys.stdout = f
        sys.stderr = f
    except Exception, e:
        print sys._stdout, str(e)

def verbose(*args):
    """prints messages to console or to log file based on the boolean options.quietmode"""
    msg = "".join([str(each) for each in args])
    print msg

def parse_nodes(cliParams):
    "parses cli node info to isi_node formatted info"
    #cmd: isi_nodes -n "1,2-5,1" %{name} %{internal}

    cmd = "isi_nodes %s %s"
    specifiers = " %{name} %{internal}"
    cliParams = ''.join(cliParams)
    exit, output = procs.get_cmd_output(cmd % (cliParams, specifiers))
    if exit:
        print "Error parsing node information: %s" % str(output)
        sys.exit(1)
    else:
        return output

##############################################################
### parse_node_ifaces function assumes the following #########
### 1. nodes_info has already been populated
### 2. There is no inconsistency between nodes specified using -c or -n
###    options and those specified in -i option. Any way, the code handling
###    nodes in -i option is not being included at this time and hence has been
###    commented out below.
def parse_node_ifaces(o, a):
    "parses cli node interfaces info"
    global myid
    myid = get_id()
#### All interfaces but no node spefied in -i option
    if a.strip().lower() == 'all':
        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()  
            SetAllifacesForNode(node_name)
#### Multiple nodes specified in -i option
#### This has been commented out for now
#    elif '+' in a:
#        multiple_nodes = a.split('+')
#        for each_node in multiple_nodes:
#            (nodeString, ifaces) = each_node.split(':')
#            if nodeString.isdigit():
#                node_name = LNNToNodeName(nodeString)
#            else:
#                node_name = nodeString
#            for each_iface in ifaces.strip().split(','):
#                if each_iface.lower() == 'all':
#                    SetAllifacesForNode(node_name)
#                    break
#                else:
#                    nic_name = get_nicname_from_node(node_name, each_iface)
#                    if nic_name != '':
#                        node_ifaces_info[node_name].append(nic_name)
#                    else:
#                        verbose( "Unable to load interface '%s'" % each_iface)
#                        sys.exit(1)
#### Single node specified in -i option
#### This has been commented out for now
#    elif ':' in a:
#        (nodeString, ifaces) = a.split(':')
#        if nodeString.isdigit():
#            node_name = LNNToNodeName(nodeString)
#        else:
#            node_name = nodeString
#        for each_iface in ifaces.strip().split(','):
#            if each_iface.lower() == 'all':
#                SetAllifacesForNode(node_name)
#                break
#            else:
#                nic_name = get_nicname_from_node(node_name, each_iface)
#                if nic_name != '':
#                    node_ifaces_info[node_name].append(nic_name)
#                else:
#                    verbose( "Unable to load interface '%s'" % each_iface)
#                    sys.exit(1)
#### Multiple interfaces but no node specified in -i option
    elif ',' in a:
        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()  
            for each_iface in a.split(','):
                if each_iface.lower() == 'all':
                    SetAllifacesForNode(node_name)
                    break
                else:
                    nic_name = get_nicname_from_node(node_name, each_iface)
                    if nic_name != '':
                        node_ifaces_info[node_name].append(nic_name)
                    else:
                        verbose( "On node %s, Unable to load interface '%s'" %(node_name,each_iface))
                        sys.exit(1)
#### Single interface specified in -i option
    else:
        iface = a.strip()
        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()  
            nic_name = get_nicname_from_node(node_name, iface)
            if nic_name != '':
                node_ifaces_info[node_name].append(nic_name)
            else:
                verbose( "On node %s, Unable to load interface '%s'" %(node_name,each_iface))
                sys.exit(1)
    return

def make_dir(logdir):
    try:
        os.makedirs(logdir)
    except OSError, e:
        if e.errno != errno.EEXIST:
            verbose("Unknown error while creating logdir")
            print "Exception:", str(e)
            sys.exit(0)
    os.system("isi set -c off " + logdir)

def sig_handler(signum, frame):
    global nodes_info, cluster_children
    global isi_netlogger_pid
    global bundle_cap_files

    curr_pid = os.getpid()
    if curr_pid == isi_netlogger_pid:
        verbose("Process interrupted by signal %s." % signum)
        terminate_procs()

        if bundle_cap_files:
            date = isys("date +\"%Y-%m-%d_%H.%M.%S\"")[0]
    
            bundledFname = 'bundled/netlog-' + date + '.tar.bz2'
            if not os.path.exists(NETLOGDIR + 'bundled'):
                 os.mkdir(NETLOGDIR + 'bundled')

            nodeDirs = ''
            for each in nodes_info:
                (nodeName, nodeAddress) = each.split()
                nodeDirs += NETLOGDIR+nodeName+" "

            cmd = "tar jcf %s %s" % (NETLOGDIR+bundledFname, nodeDirs)
            exit, output = procs.get_cmd_output(cmd)
            if exit:
                verbose( "Error bundling the Capture files located in (%s)." % NETLOGDIR) 
            else:
                verbose( "Bundling captures successful.")
                for each in nodeDirs.split():
                    if os.path.exists(each):
                        shutil.rmtree(each)
                        verbose( "Removing node capture folder: ", each)
        before_exit()
        sys.exit(0)
    else: ## signal handler was called from child process
          ## before os.forkpty is complete or has just completed
          ## and the child has not yet done execv to isi_rdo
        sys.exit(0)
        

def parse_cl_options(args):
    """removes the cli options  that are either not needed by tcpdump or passed separately"""
    isi_nlopts = ['-n:', '-t:', '-i:', '-b:', '-p', '-q', '-c', '-d', '-z']
    for each in isi_nlopts:
        opt = each.replace(':','')    #cli option without the colon 
        if opt in args:
            if each.endswith(':'): #opt and arg
                args.remove(args[args.index(opt)+1])
                args.remove(args[args.index(opt)])
            else: #just opt
                args.remove(args[args.index(opt)])

    """convert the isi_netlogger options to tcpdump options"""
    isi_nl_replaceOpts = [['-k', '-W'], ['--','']]
    for each in isi_nl_replaceOpts:
        if each[0] in args:
            args[args.index(each[0])] = each[1]

    return args

def before_exit():
    """things to do before shutting down"""
    global bpf_bufsize_specified
    if bpf_bufsize_specified:
        restore_bpf_bufsize()

    if os.path.exists(PIDFILE):
        os.remove(PIDFILE)
    if not sys.stdout.isatty():
        #quietMode was turned on. so close file to flush data
        sys.stdout.close()

def chk_lock_file():
    if not os.path.exists(LCKFILE):
        #end isi_nl reusing sighandler
        sig_handler(None,None)

def mk_lock_file():
    if not os.path.exists(LCKFILE):
        f = open(LCKFILE,'w')
        f.write('')
        f.close()

###  This function was for Logical Node number in -i option. The -n option
###  option for nodes also accepts Logical Node number as input.
###  This has been commented out as specifying node in -i option is not being included 
###  at this time.
#def LNNToNodeName(lnn):
#    cmd = "isi_nodes -n %s %s"
#    specifiers = " %{name}"
#    exit, output = procs.get_cmd_output(cmd % (lnn, specifiers))
#    if exit:
#        print "Error converting LNN to Node Name: %s" % str(output)
#        sys.exit(1)
#    else:
#        return output[0]

def NodeNameToNodeId(node_name):
    cmd = "isi_nodes -n %s %s"
    specifiers = " %{devid}"
    exit, output = procs.get_cmd_output(cmd % (node_name, specifiers))
    if exit:
        print "Error getting Node ID from Node Name: %s" % str(output)
        sys.exit(1)
    else:
        return int(output[0])

def SetAllifacesForNode(node_name):
    global myid
    global active_lnis_nics

    devid = NodeNameToNodeId(node_name)
    for lni_name, nic_name in active_lnis_nics[devid]:
        nic_name_from_node = get_nicname_from_node(node_name, nic_name)
        if nic_name_from_node != '':
            node_ifaces_info[node_name].append(nic_name_from_node)
        else:
            verbose( "On node %s, Unable to load interface '%s'" %(node_name,nic_name))
            sys.exit(1)

def GetAllActiveLnisNics():
    global active_lnis_nics
    for devid_info in ifsc.get()['node list']:
        devid = devid_info['node id']
        for lni in flx.lni_query(devid, flx.LNI_ANY_IDX, flx.IFFLAG_EXTERNAL):
            if lni.enabled and lni.nic.up and lni.nic.active:
               active_lnis_nics[devid].append((lni.name, lni.nic.name)) 


def pid_exists(pid):
    try:
        os.kill(pid, 0)
    except OSError, e:
        return False 
    else:
        return True

##############################################
### This function assumes the following:
### cluster_children (list of pid, fd tuples)
### has already been populated
def terminate_procs():
    global cluster_children
    global rem_children

    for pid, fd in cluster_children:
        ### Send KILL signal to isi_rdo pid
        if pid_exists(pid):
            ### Close pseudo terminal masterfd
            try:
                os.close(fd)
            ## this is possible if Ctrl-C was hit while this function was
            ## being executed at the end of an iteration (option -t)
            except OSError, e:
                pass
            rem_children.append(pid)
            os.kill(pid, signal.SIGKILL)

    ### Call wait for all remaining child processes together
    ### rather than waiting for a specific pid 
    t1 = time.time()
    timeout = False
    while (len(rem_children) and not timeout):
        try:
            ret_pid, status = os.waitpid(-1, os.WNOHANG)
        ## this is possible if Ctrl-C was hit while this function was
        ## being executed at the end of an iteration (option -t)
        except OSError, e:
            if e.errno == 10: ## ECHILD - no child process
                break
            else: #### Some problem with os.waitpid
                  #### but we can continue rather than exiting
                pass
        else:
            if ret_pid != 0 and ret_pid in rem_children:
                if os.WIFSIGNALED(status): ### this is expected
                    rem_children.remove(ret_pid)
                else: ### Unexpected but removing child pid any way
                    rem_children.remove(ret_pid)
            else: #### i.e. ret_pid is 0 which occurs when none of the children has terminated or a child which was not in rem_children has terminated (due to Ctrl-C near os.forkpty)
                pass

        #### This timeout condition is being used to prevent isi_netlogger from
        #### forever looping for exit status of children if for some (unexpected)
        #### reason child processes do not terminate
        t2 = time.time()
        if (t2 - t1) > 5:
            timeout = True
        else:
            timeout = False

    ### If some children pids are still remaining
    if (len(rem_children)):
#        verbose("Could not receive exit status of %d isi_rdo process(es)" %(len(rem_children)))
        pass

# Globals (mainly used by sig handler)
cluster_children = []
nodes_info = ''
node_ifaces_info = defaultdict(list) ## Node interfaces to be logged
node_bpf_bufsize = defaultdict(tuple)
rem_children = [] #### Remaining children whose exit status to wait for
isi_netlogger_pid = -1
active_lnis_nics = defaultdict(list) ## All active interfaces on all nodes 
bundle_cap_files = True

def main(argv):
    global cluster_children, nodes_info
    global node_ifaces_info
    global myid
    global logdir
    global bpf_bufsize_specified
    global isi_netlogger_pid
    global bundle_cap_files

    sleep_mins = 10
    keep_count = 3
    iface = get_default_iface()
    hostname = get_hostname()
    logdir = os.path.join(NETLOGDIR, hostname)
    snaplen = 320
    bpf_bufsize_specified = False
    print_cmd = False
    arp_included = False
    quietmode = False
    daemon = False
    ifaces_specified = False

    try:
        opts, args = getopt.getopt(argv[1:], "k:i:t:s:b:pan:chqdz")
    except getopt.GetoptError, e:
        verbose( "Invalid argument: ", e)
        syntax(logdir) 
        return

    try:
        for o, a in opts:
            if o == '-d':
                daemon = True
            elif o == "-k":
                keep_count = int(a)
            elif o == "-t":
                sleep_mins = int(a)
            elif o == '-i':
                 ifaces_specified = True
                 iface_o = o
                 iface_a = a 
            elif o == "-s":
                snaplen = int(a)
            elif o == "-h":
                syntax(logdir)
                return
            elif o == "-b":
                bpf_bufsize_specified = True
                bpf_o = o
                bpf_a = a
            elif o == "-p":
                print_cmd = True
            elif o == "-a":
                arp_included = True
            elif o == '-n':
                nodes_info = parse_nodes([o,a])
            elif o == '-c':
                nodes_info = parse_nodes('')
            elif o == '-q':
                quiet_mode()
                #options.quietmode = True
            elif o == '-z':
                bundle_cap_files = False 
    except Exception, e:
        print e
        sys.exit(1)

    if keep_count:
        verbose("Netlogger keeping %d log%s of %d minute%s each..." \
              % (keep_count, plural(keep_count),
                 sleep_mins, plural(sleep_mins)))
    else:
        verbose("Netlogger creating logs of %d minute%s each..." \
              % (sleep_mins, plural(sleep_mins)))

    signal.signal(signal.SIGINT, sig_handler)
    signal.signal(signal.SIGHUP, sig_handler)
    signal.signal(signal.SIGTSTP, sig_handler)
    signal.signal(signal.SIGTERM, sig_handler)

    if nodes_info == '':
        nodes_info = parse_nodes(['-L',' '])

    GetAllActiveLnisNics()

    if ifaces_specified:
        parse_node_ifaces(iface_o, iface_a)

    if bpf_bufsize_specified:
        process_bpf_bufsize(bpf_o, bpf_a)

    verbose("running on nodes: ", nodes_info)

    logdirs = []
    for each in nodes_info:
        name, address = each.split()
        logdir = os.path.join(NETLOGDIR, name)
        logdirs.append(logdir)
        make_dir(logdir)

    post_args = args #these are optional tcpdump args
    cmd = ['/usr/sbin/tcpdump -U'] + parse_cl_options(argv[1:])
    cmd = ' '.join(cmd).replace(' '.join(post_args),'') #rem optional args. they go at the end.
    cmd = cmd.split()
    
    if not arp_included:
        if args:
            post_args += [" and "]
        post_args += [" not arp"]

    if len(nodes_info) and os.path.exists(PIDFILE):
        print "Pid file (%s) exists. Can't run another clustered instance" % PIDFILE
        sys.exit(0)

    os.setpgrp()
    if daemon:
        daemonize()

    mk_lock_file()
    mk_pid_file()

    ### This iteration number variable is being added so that printing of command
    ### can be limited to once.
    ### In future (if required), this can be used to limit isi_netlogger to specified number of iterations
    iter_no = 1

    while 1:
        chk_lock_file()
        cluster_children = [] #replace with cluster_children
        date = isys("date +\"%Y-%m-%d_%H.%M.%S\"")[0]

        for i in range(len(nodes_info)):
            node_name, node_address = nodes_info[i].split()  
            pre_args = ['isi_rdo']+ [node_address]
            if node_ifaces_info[node_name]:
                for j in node_ifaces_info[node_name]:
                    iface_args = [' -i '] + [j] 
                    if print_cmd and iter_no == 1:
                        verbose("Running on node %s Command:" %(node_name), ' '.join(cmd + iface_args + post_args))
                    pst_args = ["-w", NETLOGDIR + "%s/netlog-%s.%s.%s.pcap" % (node_name, node_name, j, date)] + iface_args + post_args
                    try:
                        pid, fd = os.forkpty()
                    except OSError, e:
                        print "os.forkpty failed. %s" % str(e)
                        sys.exit(1)
            
                    if pid == 0:
                        os.execv('/usr/bin/isi_rdo', pre_args + cmd + pst_args)
                    cluster_children.append((pid, fd))
            else:
                if print_cmd and iter_no == 1:
                    verbose("Running on node %s Command:" %(node_name),' '.join(cmd + post_args))
                pst_args = ["-w", NETLOGDIR + "%s/netlog-%s.%s.pcap" % (node_name, node_name, date)] + post_args
                try:
                    pid, fd = os.forkpty()
                except OSError, e:
                    print "os.forkpty failed. %s" % str(e)
                    sys.exit(1)

                if pid == 0:
                    os.execv('/usr/bin/isi_rdo', pre_args + cmd + pst_args)
                cluster_children.append((pid, fd))

        time.sleep(60*sleep_mins)
        terminate_procs()

        for each in logdirs:    
            cleanup(each, keep_count)

        iter_no = iter_no + 1

if __name__ == "__main__":
    main(sys.argv)
