#! /usr/libexec/isilon/isi_eth_mixer_d

# Copyright C 2002-2008 by Isilon Systems, Inc. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
# 
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
# 
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems. 
#  
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.

'''
 $Id: isi_eth_mixer_d 192659 2011-07-08 16:46:55Z olaskin $
'''

import string
import os
import select
import signal
import sys
import time
import traceback

import isi.app.daemon

from isi.net import route
from isi.net import ping
from isi.fs.restripe.util import setSigHandlers, setSigHandlersDefault
from isi.app.lib.process import madvise, MADV_PROTECT

from   isi.net.ethmixer.node   import Node, node_set_verbosity
from   isi.net.ethmixer.iface  import Interface
from   isi.net.ethmixer.dest   import ALIVE, DOWN, DEAD, uidassoc
import isi.net.ethmixer.net    as     net
from   isi.net.ethmixer.net    import Network
from   isi.net.ethmixer.config import conf, config_get
from   isi.net.ethmixer.layout import layout
import isi.net.ethmixer.log    as     log
from   isi.net.ethmixer.util   import timeleft
import isi.net.ethmixer.util   as     util

import isi.app.lib.cluster as cluster
import isi.net.flexnet as flx

debug = 0

# (non)running codes
KEEP_RUNNING = 0
SHUTDOWN_GRACEFUL = 1
SHUTDOWN = 2

################ EthMixer Daemon ################

class EthMixerDaemon:
    '''Daemon to manage intra-cluster routes for load balancing and failover.'''

    # icg                      c    config_get()
    # rc                       c    EthMixerConfig
    # Network networks[]       c    len(flx.internal_subnet_load())
    # Interface interfaces[]   c    len(flx.internal_subnet_load())
    # Node nodes[]             c    numnodes
    # num_nodes                c    min(maximum offset of configured node's into
    #                               primary interface's ip list + ROUTE_AHEAD ,
    #                               the lowest IP address list)
    # mynodeid                 c    icg['current node id']
    # usenet                   c    networks[idx] where flags has USEADDR set
    # usenetidx                c    idx of usenet
    # internalnets             c    networks[*] where flags has INTERNAL set
    #
    # fd route_fd              r    route state
    # fd ping_fd               r    ping state
    # ping                     r    ping object to receive pings
    # needReconfig             r    sig HUP
    # wasfast                  r    ping state, last fast ping
    # wasslow                  r    ping state, last slow ping
    # wasroute                 r    ping state, last route checkup

    def __init__(self):
        self.done = 0
        self.do_dump_state = 0
        self.start_time = time.time()

        if os.path.basename(sys.argv[0]) == 'python':
            self.name = 'isi_eth_mixer_d'
        else:
            self.name = os.path.basename(sys.argv[0])

        self.log_file = log.open(self.name, debug)
        conf.read(debug)

        # Set signal handlers.
        setSigHandlers({
            signal.SIGHUP  : self.handleSigHup,
            signal.SIGTERM : self.handleSigTerm,
            signal.SIGINT  : self.handleSigInt,
            signal.SIGUSR1 : self.handleSigUsr1,
            signal.SIGUSR2 : self.handleSigUsr2,
        })

        # Daemonize.
        if not debug:
            isi.app.daemon.startstop()


    def main(self):
        log.info("starting up.")

        madvise(MADV_PROTECT)

        if not os.path.exists("/etc/ifs/array.xml"):
            log.info("waiting for cluster config")
            while not os.path.exists("/etc/ifs/array.xml"):
                time.sleep(600)

        self.init()
        self.running = KEEP_RUNNING

        while self.running == KEEP_RUNNING:
            while not self.configOk():
                log.info("waiting for cluster config requiring routing.")
                self.needReconfig = 0
                while not self.needReconfig:
                    # wait for signal
                    time.sleep(600)

                self.check_stop()
                self.icg = config_get()

            self.re_init()

            # restart timers
            self.wasfast = self.wasslow = self.wasroute = time.time()
            self.wascheck_active = self.wasfast

            while not self.needReconfig:
                self.main_loop()
                self.check_stop()

            self.needReconfig = 0
            self.icg = config_get()

        self.check_stop()
        sys.exit(0)

    def recv_pings(self):
        rcvd = self.ping.recv()
        if rcvd < 0:
            log.warn("ping.recv() failed: %d" % rcvd)

    def main_loop(self):
        now = time.time()

        # reset our times if now is wacky
        if now < self.wasfast or now > (self.wasroute + 10 * conf.route_poll):
            self.wasfast = self.wasslow = self.wasroute = now
            self.wascheck_active = self.wasfast

        left = timeleft(now, self.wasfast + conf.fast_poll)

        try:
            [infds, outfds, exfds] = select.select(
                    [self.ping_fd], [], [], left)
        except:
            return

        now = time.time()

        # Receive any outstanding pings from the previous iteration.
        if self.ping_fd in infds:
            self.recv_pings()

        # ping stuff
        if timeleft(now, self.wasfast + conf.fast_poll) == 0:
            self.wasfast = now
            self.check_interfaces()
            for node in self.nodes:
                node.do_ping(active=1)
            self.recv_pings()

        if timeleft(now, self.wasslow + conf.slow_poll) == 0:
            self.wasslow = now
            for node in self.nodes:
                node.do_ping(active=0)
            self.recv_pings()

        # change routes according to new ping state
        self.do_statechanges()

        # occasionally check that routes are still valid.
        if timeleft(now, self.wasroute + conf.route_poll) == 0:
            self.wasroute = now
            for node in self.nodes:
                node.check_route()
   
        # occasionally do a bug check
        if timeleft(now, self.wascheck_active + conf.check_active_poll) == 0:
            self.wascheck_active = now
            for node in self.nodes:
                node.check_active()

        if self.do_dump_state:
            self.do_dump_state = 0
            self.dump_state()


    ################ init ################
        
    def init(self):
        self.needReconfig = 0
        self.icg = config_get()
        self.local = cluster.Device(self.icg['current node id'])
        self.subnets = flx.internal_subnet_load()
        
        self.mynodeid = self.local.devid
  
        # set up internal networks
        self.networks = []
        self.interfaces = []
        for netidx in range(0, len(self.subnets)):
            self.networks += [Network(self.subnets[netidx], netidx)]
            interface = self.local.interface(self.subnets[netidx].flags,
                    self.subnets[netidx].index)
            ip = util.ipaddress(self.subnets[netidx], self.local.devid)
            self.interfaces += [Interface(interface, ip,
                    self.networks[netidx])]
  
        self.num_nodes = 0
  
        # wait for re_init() before configuring nodes
        self.nodes = []

        self.init_ping()
        self.init_route()


    def init_ping(self):
        self.ping = ping.create(255)
        self.ping.set_opts(['ping', '127.0.0.1'])
        self.ping_fd = self.ping.socket()

        if not debug:
            #ping.set_logging_fd(self.log_file)
            pass


    def init_route(self):
        route.init()
        route.set_verbosity(conf.verbose)
        self.route_fd = route.get_fd()

        # Messages from the route library are useful to have.
        route.set_logging_fd(self.log_file)


    def re_init(self):
        self.needReconfig = 0

        old_icg = self.icg
        old_local = self.local
        
        self.icg = config_get()
        self.local = cluster.Device(self.icg['current node id'])

        # reread our main configuration
        conf.read()

        # find my interfaces
        new_subnets = flx.internal_subnet_load()
 
        # first exclude things we can't handle...
        if len(new_subnets) != len(self.networks):
            log.warn("reconfig: cannot handle changing number of networks. "
                    "restarting")
            self.shutdown("reconfig failure, number of networks changed")
  
        if len(new_subnets) != len(self.interfaces):
            log.warn("reconfig: cannot handle changing number of "
                    "interfaces. restarting")
            self.shutdown("reconfig failure, number of interfaces changed")
  
        if self.mynodeid != self.local.devid:
            log.warn("reconfig: cannot handle changing my node id. "
                    "restarting")
            self.shutdown("reconfig failure, my node id changed")
  
        self.subnets = new_subnets

        # find internal and use networks
        self.parse_net_flags()
  
        # re-configure networks and interfaces
        for netidx in range(0, len(self.subnets)):
            self.networks[netidx].reconfig(self.subnets[netidx])
            ip = util.ipaddress(self.subnets[netidx], self.local.devid)
            self.interfaces[netidx].reconfig(ip)
  
        # reconfigure nodes, less one as we won't create a node for
        # ourselves.
        new_num_nodes = len(self.icg['node list'] ) - 1

        #
        # change/add/delete configured nodes
        #
        newnodes = []
        for nodeconf in self.icg['node list']:
            # Skip local node.
            if nodeconf['node id'] == self.local.devid:
                continue

            dev = cluster.Device(nodeconf['node id'])

            # build happy config structure to make node's life easy.
            nodeinfo = self.happy_nodeinfo(dev)
            useip = nodeinfo['useip']

            node = self.find_nodeid(self.nodes, nodeconf['node id'])

            # Sanity check.
            try:
                self.usenet.ips.index(useip)
            except:
                log.warn("reconfig: illegal config, node %d: %s not in "
                        "%s's IPs" % (nodeconf['node id'], useip,
                        self.networks[self.usenetidx].name))
                continue

            # Reconfig if existing node or create a new Node object.
            if node != None:
                node.reconfig(nodeinfo, self.interfaces)
            else:
                node = Node(nodeinfo, self.interfaces)

            newnodes += [node]

        # for nodes being deleted, delete their routes.
        for node in self.nodes:
            if self.find_nodeid(newnodes, node.nodeid) is None:
                node.do_delete_route()

        self.nodes = newnodes
        self.num_nodes = new_num_nodes

        self.do_layout()
        self.check_interfaces()
        self.do_statechanges()


    # get our node information straight from configuration.
    def happy_nodeinfo(self, nodeconf):
        nodeinfo = { 'node id': nodeconf['node id'] }
        useip = util.ipaddress(self.usenet.subnet, nodeconf['node id'])
        nodeinfo['useip'] = useip
        nodeinfo['usenet'] = self.usenet

        nodeinfo['internal'] = []

        for net in self.internalnets:
            nodeinfo['internal'].append([util.ipaddress(net.subnet,
                    nodeconf['node id']), net])
        return nodeinfo


    ################ helpers ################

    def do_layout(self):
        l = layout(self.icg, self.subnets, conf.verbose)

        for node in self.nodes:
            if l.has_key(node.nodeid):
                node.set_primary(l[node.nodeid])
            elif node.nodeid != self.mynodeid and node.nodeid != -1:
                log.warn("layout: does not contain node %d" % node.nodeid )
                log.warn("layout: " + str(l))
                l = layout(self.icg, self.subnets, 1)


    def check_interfaces(self):
        for iface in self.interfaces:
            if iface.check_interface():
                for node in self.nodes:
                    node.interface_change(iface)

    
    def do_statechanges(self):
        for node in self.nodes:
            if node.statechange:
                log.info("doing state changes for node %d" % node.nodeid)
                node.do_statechange()
                node.statechange = 0


    def parse_net_flags(self):
        self.internalnetidxs = net.find_internalnetidxs(self.subnets)
        [tmp, self.usenetidx] = net.find_usenet(self.subnets)

        self.internalnets = []
        for idx in self.internalnetidxs:
            self.internalnets += [self.networks[idx]]

        if self.usenetidx != None:
            self.usenet = self.networks[self.usenetidx]
    

    def configOk(self):
        self.parse_net_flags()
        return (self.usenetidx != None and self.internalnetidxs != None)
    

    def find_nodeid(self, nodes, nodeid):
        for node in nodes:
            if node.nodeid == nodeid:
                return node
      
        return None
    

    def dump_state(self):
        flags = { flx.NETFLAG_USEADDR: "use",
                  flx.NETFLAG_DISABLED:"disabled",
                  flx.IFFLAG_INTERNAL:"int" }
        state = { ALIVE: "alive",
                  DOWN:  "down",
                  DEAD:  "dead" }

        if conf.verbose:
            log.info("my id:%d  routefd:%d  pingfd:%d" % (self.mynodeid,
                    self.route_fd, self.ping_fd))

            log.info("nets:")
            for net in self.networks:
                s = "  %s(%d) " % (net.name, self.networks.index(net))
                if net.idx != self.networks.index(net):
                    s += "(net contains idx of %d)" % net.idx
                s += "ips[" + string.join(net.ips) + "] "

                s += "flags[ "
                for flag in flags:
                    if net.flags & flag:
                        s += flags[flag] + " "
                s += "] "

                if net == self.usenet:
                    s += "usenet "
                    if net.idx != self.usenetidx:
                        s += "(not usenetidx %d)" % self.usenetidx

                if net in self.internalnets:
                    s += "internalnet"

                log.info(s)

            log.info("interfaces:")
            for iface in self.interfaces:
                s = "  %s(%d) " % (iface.name, self.interfaces.index(iface))
                s += "net(%s) " % iface.net.name
                s += "ip[" + iface.ip + "] "
                if iface.active:
                    s += "(active)"
                else:
                    s += "(inactive)"

                log.info(s)

        log.info("nodes:")
        if self.num_nodes != len(self.nodes):
            log.info("  num_nodes(%d) != len(nodes)(%d)" %
                  (self.num_nodes, len(self.nodes)))


        for node in self.nodes:
            s = "  node %d: " % node.nodeid
            s += "[ %s -> %s ] " % (node.router.destination(),
                    node.router.gateway())
            if not node.configed:
                s += "(unconfigured) "
            (exists, ip) = node.router.get()
            if exists != 1 or node.router.gateway() != ip:
                if exists != 1:
                    s += "!!!route doesn't exist!!!"
                else:
                    s += "!!!-> %s !!!" % ip

            if node.statechange:
                s += "(statechange pending)"
            log.info(s)

            for dest in node.destinations:
                s = "    %s " % dest.iface.name
                s += "--> %s " % dest.pinger.destination()

                if dest == node.primary:
                    s += "primary "

                if dest == node.active:
                    s += "active"
                    
                log.info(s)

                s = "            "
                s += "id:%2d " % dest.uid
                s += "bad:%2d " % dest.bad_count
                s += "good:%2d " % dest.good_count
                s += "tx:%2d " % dest.pinger.tx_count()
                s += "rx:%2d " % dest.pinger.rx_count()
                if dest.pinger.time_sent() != 0:
                    s += "sent:%3.3f " % (dest.pinger.time_sent() -
                            self.start_time)
                if dest.pinger.time_recv() != 0:
                    s += "recv:%3.3f " % (dest.pinger.time_recv() -
                            self.start_time)
                s += state[dest.alive] + " "

                log.info(s)

        if conf.verbose:
            log.info("Dest ID Allocator: avail %d alloced %d" %
                     (len(uidassoc.available_ids),
                      len(uidassoc.alloced_ids)))


    ################ shutdown ################

    # called by main to avoid error checking.
    def check_stop(self):
        if self.running != KEEP_RUNNING:
            # Set handlers back to default
            setSigHandlersDefault([signal.SIGHUP, signal.SIGTERM,
                    signal.SIGINT, signal.SIGUSR1, signal.SIGUSR2])

            if self.running == SHUTDOWN_GRACEFUL:
                # clean up routes
                for node in self.nodes:
                    node.do_delete_route()

            sys.exit(0)


    # shutdown is called from interrupt (or not)
    def shutdown(self, reason=None, graceful=SHUTDOWN):
        if reason:
            log.notice("Shutting down due to \"%s\"" % reason)

        # Set handlers back to default
        setSigHandlersDefault([signal.SIGHUP, signal.SIGTERM,
                signal.SIGINT, signal.SIGUSR1, signal.SIGUSR2])

        self.running = graceful
        sys.exit(0)


    ################ signal ################

    def handleSigHup(self, signum, frame):
        self.needReconfig = 1
        log.close()
        self.log_file = log.open(self.name, debug)
        route.set_logging_fd(self.log_file)

    def handleSigTerm(self, signum, frame):
        self.shutdown("SIGTERM")

    def handleSigInt(self, signum, frame):
        self.shutdown("SIGINT", graceful=SHUTDOWN_GRACEFUL)

    def handleSigUsr1(self, signum, frame):
        '''USR1 dumps information'''
        log.info("Received SIGUSR1, dumping state.")
        self.do_dump_state = 1

    def handleSigUsr2(self, signum, frame):
        '''USR2 toggles verbosity level'''
        if conf.verbose:
            log.info("Received SIGUSR2, verbosity disabled")
            conf.verbose = 0
            node_set_verbosity(0)
            route.set_verbosity(0)
        else:
            log.info("Received SIGUSR2, verbosity enabled")
            conf.verbose = 1
            node_set_verbosity(1)
            route.set_verbosity(1)

def main():
    global debug
    
    if len(sys.argv) == 2:
        icg = config_get()
        subnets = flx.internal_subnet_load()

        if sys.argv[1] == 'testconfig':
            conf.read()
            conf.show()
            internalnets = net.find_internalnets(subnets)
            [ usenet, tmp ] = net.find_usenet(subnets)
            if usenet != None and internalnets != None:
                print("System config is good for routing.")
            else:
                reason = "missing "
                if usenet == None:
                    reason += "USEADDR"
                if internalnets == None:
                    reason += "INTERNAL"
                print "System config is insufficient for " \
                        "routing [%s].\n" % reason

            sys.exit(0)

        elif sys.argv[1] == 'showlayout':
            layout(icg, subnets, 1)
            sys.exit(0)

        elif sys.argv[1] == 'showprimary':
            [ usenet, usenetidx ] = net.find_usenet(subnets)
            if usenetidx == None:
                print "Cluster config insufficient for routing.  No " \
                        "active routes."
                sys.exit(0)

            dict = layout(icg, subnets, 0)
            print "primary: %2d ->" % icg['current node id']
            for nodeconf in icg['node list']:
                id = nodeconf['node id']
                if dict.has_key(id):
                    name = subnets[dict[id]].internal_lni_name
                    print " %2d:%s" % (id, name)
                else:
                    print " !!:!!!!"
            print "\n"
            sys.exit(0)

        elif sys.argv[1] == 'showactive':
            [ usenet, usenetidx ] = net.find_usenet(subnets)
            if usenetidx == None:
                print "Cluster config insufficient for routing.  No " \
                        "active routes."
                sys.exit(0)

            ips = util.get_ips(usenet, conf.max_ips)
            s = ""
            for ip in ips:
                s += "%s \n" % ip
            os.system("/usr/bin/netstat -rn | /usr/bin/grep -F '%s'" % s)
            sys.exit(0)

        elif sys.argv[1] == 'debug':
            debug = 1

    elif len(sys.argv) == 3 and sys.argv[1] == 'sig':
        pidfile = isi.app.daemon.PidFile(None)
        pid = pidfile.getpid()
        if not pid:
            print "error: pid file '%s' missing.\n" % pidfile.name()
        else:
            os.system("kill -%s %d" % (sys.argv[2], pid))
        sys.exit(0)

    if isi.app.daemon.validAction() or debug == 1:
        emd = EthMixerDaemon()
        try:
            emd.main()
        except Exception, e:
            if not isinstance(e,SystemExit):
                if not debug:
                    traceback.print_exc(file=emd.log_file)
                else:
                    traceback.print_exc()
    else:
        print("Usage: %s start|stop|restart|debug  (debug doesn't fork)"
              % sys.argv[0])
        print("Internal use only: testconfig|showlayout|showprimary|"
                "showactive")
        sys.exit(0)

if __name__ == '__main__':
    main()

# vim:set filetype=python:
