#!/usr/libexec/isilon/isi_esrs_d
"""isi_esrs_d ESRS service to manage IP updates and periodic requests"""

import errno
import json
import os
import select
import socket
import struct
import sys
import time
import threading
from threading import Timer
import traceback

import isi.app.lib.cluster as cluster
import isi.celog
from isi.celog.eventids import eventid_map
import isi.config.licensing as licensing
import isi.config.tardis as tardis
import isi.daemon.daemon as daemon
import isi.esrs.restclient as esrs
import isi.gconfig as gconfig
import isi.net.flexnet as flexnet
import isi.esrs.usage_intel as ui
from isi.esrs.ilogger import Ilogger

ISI_ESRS_DAEMON_NAME = "isi_esrs_d"
# flag for CELOG to determine eligibility to send alerts
ESRS_ELIGIBLE_FLAG = "/etc/ifs/esrs_eligible"
GCONFIG_ESRS_TREE_NAME = "esrs"
TARDIS_GCONFIG_ESRS_NS = "Gconfig.esrs_config"
FLEXNET_CONFIG_XML_FILENAME = "/etc/ifs/flx_config.xml"
TARDIS_REGISTERED_NODE_NS = "esrs.registered.state"

LOG = None
CONFIG = None
ATTEMPTS = 3 # number of transaction attempts before exit
CONNECTED = None

def init_logging():
    "Intialise logging facilities"
    global LOG
    LOG = Ilogger(ISI_ESRS_DAEMON_NAME)

def log_exc(exc_type, value, tb):
    LOG.error("Encountered uncaught exception: {e} {v} {tb}"
              .format(e=exc_type.__name__,
                      v=value,
                      tb=traceback.format_tb(tb)))


class ESRSSockWatchee(object):
    """Helper class to watch a socket (e.g., Tardis subscriptions)"""
    def __init__(self, item, name):
        LOG.debug("ESRSSockWatchee.__init__ with name {n}"
                  .format(n=name))
        self.name = name
        self.sock = item
        self.kevent = select.kevent(item, filter=select.KQ_FILTER_READ,
                                    flags=(select.KQ_EV_ADD
                                           | select.KQ_EV_ENABLE))

    def __del__(self):
        LOG.debug("ESRSSockWatchee.__del__ with name {n}"
                  .format(n=self.name))
        self.sock.close()

    def fileno(self):
        """Return the file descriptor (int) associated with this watchee."""
        LOG.debug("ESRSSockWatchee.fileno with name {n}"
                  .format(n=self.name))
        return self.sock.fileno()

    def recv(self, msg_size):
        """Receive data from watched socket."""
        LOG.debug("ESRSSockWatchee.recv with name {n}"
                  .format(n=self.name))
        recvd = ""
        while len(recvd) < msg_size:
            msg = self.sock.recv(msg_size - len(recvd))
            if len(msg) == 0:
                return None
            recvd += msg
        LOG.debug("finished ESRSSockWatchee.recv with name {n}"
                  .format(n=self.name))
        return recvd

    def drain_sock(self):
        """Empty watched socket."""
        LOG.debug("ESRSSockWatchee.drain_sock with name {n}"
                  .format(n=self.name))
        msg_len_bytes = self.sock.recv(struct.calcsize('Q'))
        msg_len, = struct.unpack('Q', msg_len_bytes)
        self.recv(msg_len)
        LOG.debug("finished ESRSSockWatchee.drain_sock with name {n}"
                  .format(n=self.name))

    def on_change(self):
        """Actions to perform on change notification"""
        LOG.debug("ESRSSockWatchee.on_change with name {n}"
                  .format(n=self.name))
        LOG.info("Registered change in namespace {f}"
                 .format(f=self.name))
        try:
            self.drain_sock()
        except socket.error:
            pass
        except struct.error:
            pass

        LOG.debug("finished ESRSSockWatchee.on_change with name {n}"
                  .format(n=self.name))


class ESRSFileWatchee(object):
    """Helper class to watch a file"""
    def __init__(self, item):
        LOG.debug("ESRSFileWatchee.__init__ with name {n}"
                  .format(n=item))
        self.name = item
        self.file = None
        self.kevent = None
        self.gen_kevent()

    def __del__(self):
        LOG.debug("ESRSFileWatchee.__del__ with name {n}"
                  .format(n=self.name))
        os.close(self.file)

    def gen_kevent(self):
        """Generate a kevent for this file"""
        LOG.debug("ESRSFileWatchee.gen_kevent with name {n}"
                  .format(n=self.name))
        self.file = os.open(self.name, os.O_RDONLY)
        self.kevent = select.kevent(self.file,
                                    filter=select.KQ_FILTER_VNODE,
                                    flags=(select.KQ_EV_CLEAR
                                           | select.KQ_EV_ADD
                                           | select.KQ_EV_ENABLE),
                                    fflags=(select.KQ_NOTE_WRITE
                                            | select.KQ_NOTE_EXTEND
                                            | select.KQ_NOTE_ATTRIB
                                            | select.KQ_NOTE_DELETE
                                            | select.KQ_NOTE_LINK
                                            | select.KQ_NOTE_REVOKE
                                            | select.KQ_NOTE_RENAME))

    def fileno(self):
        """Return the file descriptor (int) associated with this watchee."""
        LOG.debug("ESRSFileWatchee.fileno with name {n}"
                  .format(n=self.name))
        return self.file.fileno()

    def on_change(self):
        """Actions to perform on change notification"""
        LOG.info("Registered change in file {f}"
                 .format(f=self.name))
        os.close(self.file)
        self.gen_kevent()
        return self

class ESRSEventWatcher(object):
    """Class to watch a group of sockets or files using kqueue"""
    def __init__(self, watchees):
        LOG.debug("ESRSEventWatcher.__init__")
        self.queue = select.kqueue()
        self.watchees = []
        self.kevents = []
        self.ident = {}

        for watchee in watchees:
            self.add(watchee)

    def __del__(self):
        LOG.debug("ESRSEventWatcher.__del__")
        self.queue.close()

    def add(self, watchee):
        """Add watchee to this event queue"""
        LOG.debug("ESRSEventWatcher.add: {n}".format(n=watchee.name))
        self.ident[watchee.kevent.ident] = watchee
        self.watchees.append(watchee)
        self.kevents.append(watchee.kevent)
        self.queue.control(self.kevents, 0, 0)
        LOG.debug("finished ESRSEventWatcher.add: {n}".format(n=watchee.name))

    def remove(self, ident):
        """Remove watchee from this event queue"""
        watchee = self.ident[ident]
        LOG.debug("ESRSEventWatcher.remove {n}".format(n=watchee.name))
        del self.ident[ident]
        self.watchees.remove(watchee)
        self.kevents.remove(watchee.kevent)
        self.queue.control(self.kevents, 0, 0)
        LOG.debug("finished ESRSEventWatcher.remove: {n}"
                  .format(n=watchee.name))

    def map_kevent_to_object(self, kevents):
        """Map kevent back to object using ident"""
        LOG.debug("ESRSEventWatcher.map_kevent_to_object")
        ids = set([kevent.ident for kevent in kevents])
        LOG.debug("ESRSEventWatcher.map_kevent_to_object")
        return [self.ident[ident] for ident in ids]

    def wait_on_change(self, timeout=None):
        """Wait on watchee changes"""
        LOG.debug("ESRSEventWatcher.wait_on_change")
        try:
            return self.map_kevent_to_object(self.queue.control([], 10000,
                                                                timeout))
        except OSError as err:
            if err.errno != errno.EINTR:
                raise err
            return []
        except Exception as err:
            LOG.error(str(err))
            LOG.debug(traceback.format_exc())
            raise err

class ESRSConfig(object):
    """ESRS configuration class"""
    def __init__(self):
        LOG.debug("ESRSConfig.__init__")
        self.lnn = None
        self.ip = None
        self.enabled = False
        self.esrs_gateways = None
        self.esrs_pools = None
        self.ui_reporting_period = None
        self.connectivity_check_period = None
        self.alert = True
        self.fetch_esrs_config()
        self.fetch_local_node_info()

    def fetch_local_node_info(self):
        """Fetch information about current running node"""
        LOG.debug("ESRSConfig.fetch_local_node_info")
        self.lnn = cluster.array.getLocalNode().lnn()

        # XXX: if pool doesn't exist this will crash
        clus = cluster.Cluster()
        flexcfg = flexnet.config_try_load()

        esrs_pools = (flexcfg.find_subnet_by_name(e['subnet'])
                      .find_pool_by_name(e['pool'])
                      for e in self.esrs_pools)
        sorted_pools = sorted(esrs_pools, key=lambda p: -1*p.is_static_pool())
        matching_devid = ((p, iid) for p in sorted_pools
                          for iid in p.iface_ids
                          if iid.devid == clus.lnn_to_devid(self.lnn))

        try:
            pool, iid = next(matching_devid)
            self.ip = pool.get_addrs(iid)[0]
        except StopIteration:
            self.ip = None
        except Exception:
            self.ip = None
        LOG.debug("finished ESRSConfig.fetch_local_node_info")

    def fetch_esrs_config(self):
        """Fetch ESRS configuration information"""

        LOG.debug("ESRSConfig.fetch_esrs_config")
        tree = gconfig.Tree("esrs")
        tree_open = gconfig.open(tree)
        ctx = tree_open.ctx_new()
        esrs_cfg = ctx.read("esrs")

        # threads should not attempt to connect to the gateway before ESRS
        # service is designated enabled
        self.enabled = esrs_cfg['enabled']
        self.alert = esrs_cfg['alert_on_disconnect']
        self.esrs_pools = esrs_cfg['gateway_access_pools']
        self.esrs_gateways = [esrs_cfg['primary_esrs_gateway'],
                                     esrs_cfg['secondary_esrs_gateway']]
        self.ui_reporting_period = esrs_cfg['ui_reporting_period']
        self.connectivity_check_period = esrs_cfg['connectivity_check_period']

        LOG.debug("finished ESRSConfig.fetch_esrs_config")

    def reload(self):
        """Refresh the configuration"""
        LOG.debug("ESRSConfig.reload")
        self.fetch_esrs_config()
        self.fetch_local_node_info()

    def check_eligibility(self):
        """Check if current node is still eligible for mastership"""
        LOG.debug("ESRSConfig.check_eligibility")
        clus = cluster.Cluster()
        flexcfg = flexnet.config_try_load()

        esrs_pools = (flexcfg.find_subnet_by_name(e['subnet'])
                      .find_pool_by_name(e['pool'])
                      for e in self.esrs_pools)
        matching_devid = ((p, iid) for p in esrs_pools
                          for iid in p.iface_ids
                          if iid.devid == clus.lnn_to_devid(self.lnn))
        try:
            pool, iid = next(matching_devid)
            return self.ip in pool.get_addrs(iid)
        except StopIteration:
            return False
        except Exception:
            return False

def remove_esrs_eligible_flag():
    """Remove eligibility flag used by CELOG to determine available nodes"""
    try:
        os.unlink(ESRS_ELIGIBLE_FLAG)
    except OSError as err:
        # not necessarily an error, just means the file didn't exist
        LOG.error(str(err))

def get_tardis_ns_subscription(ns):
    """Get a subscription to a tardis namespace"""
    LOG.debug("get_tardis_ns_subscription")
    ns = tardis.Namespace(ns, tardis.TNS_O_CREAT)
    return ns.subscribe()

def get_watchee_list(namespaces, filenames):
    """Create a list of watchees from tns and filenames"""
    LOG.debug("get_watchee_list")
    watchees = []

    for ns in namespaces:
        try:
            sock_sub = get_tardis_ns_subscription(ns)
        except tardis.TardisError as err:
            LOG.error(str(err))
            LOG.debug(traceback.format_exc())
            raise err

        sock_watchee = ESRSSockWatchee(sock_sub, ns)
        watchees.append(sock_watchee)

    for filename in filenames:
        file_watchee = ESRSFileWatchee(filename)
        watchees.append(file_watchee)

    LOG.debug("finished get_watchee_list")
    return watchees

def wait_on_kevent_watcher(watcher):
    """Wait on changes to kevents"""
    LOG.debug("wait_on_kevent_watcher")
    changed_watchees = watcher.wait_on_change()
    for watchee in changed_watchees:
        new_watchee = watchee.on_change()
        if new_watchee:
            watcher.remove(watchee.kevent.ident)
            watcher.add(new_watchee)
    LOG.debug("finished wait_on_kevent_watcher")

def esrs_celog_event(add):
    """Create a celog alert when disconnected from ESRS"""
    if not CONFIG.alert:
        return

    LOG.debug("esrs_celog_event")
    specifier = {'job': ISI_ESRS_DAEMON_NAME}
    esrsev = isi.celog.Event(
        event_id=eventid_map["SW_ESRS_DISCONNECTED"],
        severity=isi.celog.EVENT_LEVEL_CRIT,
        message=("{a} attempts to contact the configured ESRS gateway(s) failed"
                 .format(a=ATTEMPTS)),
        start_time=int(time.time()),
        value=10,
        specifier=specifier
    )

    if add:
        LOG.info("Trying to add ESRS disconnected event...")
        try:
            esrsev.add()
        except SystemError as err:
            LOG.error("Error while sending CELOG ESRS event: {s}"
                      .format(s=str(err)))
            raise err
        else:
            if esrsev.error_code:
                LOG.error("CELOG returned error {d} while processing ESRS event"
                          .format(d=esrsev.error_code))
                raise SystemError("CELOG returned error {d} while processing"
                                  " ESRS event".format(d=esrsev.error_code))
            else:
                LOG.info("CELOG event id {i} sent with severity {s}"
                         .format(i=esrsev.event_id, s=esrsev.severity))
    else:
        LOG.info("Trying to end ESRS disconnected event...")
        try:
            esrsev.add()
            esrsev.end()
        except SystemError as err:
            LOG.info("Failed to end CELOG event id {i} sent with severity {s}"
                     "and message {m}: {e}".format(i=esrsev.event_id,
                                                   s=esrsev.severity,
                                                   m=esrsev.message,
                                                   e=str(err)))
        else:
            if esrsev.error_code:
                LOG.error("CELOG returned error {d} while processing ESRS event"
                          .format(d=esrsev.error_code))
                raise SystemError("CELOG returned error {d} while processing"
                                  " ESRS event".format(d=esrsev.error_code))
            else:
                LOG.info("CELOG event id {i} ended"
                         .format(i=esrsev.event_id))
    LOG.debug("finished esrs_celog_event")

def esrs_get_status():
    """Get connectivity status to gateway"""
    global CONNECTED
    # only attempt the transaction if ESRS is enabled
    if CONFIG.enabled:
        LOG.info("Requesting connectivity status from ESRS...")
        swid = licensing.get_swid()
        # retry ATTEMPTS times
        for attempt in range(ATTEMPTS):
            try:
                status = esrs.pcm_get_connectivity_status(swid)
            except esrs.ESRSError as err:
                LOG.error("Get connectivity status attempt {a} failed: {err}"
                          .format(a=attempt, err=str(err)))
                LOG.debug(traceback.format_exc())
            else:
                status = json.loads(status)
                LOG.info("Connectivity status: {s}".
                         format(s=status["deviceStatus"]))
                # only try to send a CELOG event if you are changing state
                # this request was successful, so we only care if we were
                # previously disconnected or this is the first request
                if CONNECTED is None or CONNECTED is False:
                    esrs_celog_event(False)
                    CONNECTED = True
                break
        else:
            LOG.error(("{a} attempts to fetch connectivity status from ESRS "
                       "backend failed, sending event and exiting..."
                       .format(a=ATTEMPTS)))
            # this request was unsuccessful ATTEMPTS times, so we will send an
            # event if we were previously connected or this is the first request
            if CONNECTED is None or CONNECTED is True:
                esrs_celog_event(True)
                CONNECTED = False
            remove_esrs_eligible_flag()
            # XXX: this isn't a very nice way to do this, but this thread needs
            # to kill the whole program when it's disconnected
            os._exit(1)
    else:
        LOG.debug("ESRS not yet enabled, waiting to get connectivity status...")

def esrs_usage_intelligence():
    """Perform usage intelligence library call"""
    if CONFIG.enabled:
        ui.send_usage_intelligence(LOG)
    else:
        LOG.debug("ESRS not yet enabled, waiting to send UI data...")

def esrs_ip_update():
    """Update registered IP on ESRS backend"""
    global CONNECTED
    # if ESRS is not yet enabled, this service may have been started
    # as part of the enablement process, but it should not attempt to
    # contact the ESRS backend until the cluster is properly registered
    ip = str(CONFIG.ip)
    if CONFIG.enabled:
        LOG.info("Updating ESRS landing IP to... {ip}"
                 .format(ip=ip))
        swid = licensing.get_swid()
        for attempt in range(ATTEMPTS):
            try:
                esrs.pcm_update_device_ip(swid, ip)
            except esrs.ESRSError as err:
                LOG.error("Update landing IP attempt {a} failed: {err}"
                          .format(a=attempt, err=str(err)))
                LOG.debug(traceback.format_exc())
            else:
                LOG.info("Successfully relayed new landing IP {ip} to ESRS"
                         " backend".format(ip=ip))
                if CONNECTED is None or CONNECTED is False:
                    esrs_celog_event(False)
                    CONNECTED = True
                break
        else:
            LOG.error("{a} attempts to relay new landing IP {ip} to ESRS "
                       "backend failed, sending event and exiting..."
                       .format(a=ATTEMPTS, ip=ip))
            if CONNECTED is None or CONNECTED is True:
                esrs_celog_event(True)
                CONNECTED = False
            remove_esrs_eligible_flag()
            exit(1)
    else:
        LOG.debug("ESRS not fully enabled, can't update IP")

def ui_thread():
    """Threading timer for Usage Intelligence calls"""
    LOG.debug("ui_thread")
    esrs_usage_intelligence()
    ui_timer = Timer(CONFIG.ui_reporting_period, ui_thread)
    ui_timer.daemon = True
    ui_timer.start()

def status_thread():
    """Threading timer for get connectivity status calls"""
    LOG.debug("status_thread")
    esrs_get_status()
    status_timer = Timer(CONFIG.connectivity_check_period, status_thread)
    status_timer.daemon = True
    status_timer.start()

def update_tardis_registered_node(ns):
    """Update/create tardis database with registered node configuration"""
    LOG.debug("update_tardis_registered_node")
    kvs = tardis.KVSet()
    kvs["esrs_registered_lnn"] = str(CONFIG.lnn)
    kvs["esrs_registered_ip"] = str(CONFIG.ip)
    try:
        ns.create(kvs)
    except tardis.TardisError as err:
        LOG.error(str(err))
        LOG.debug(traceback.format_exc())

    while True:
        ns.sync(kvs)
        kvs["esrs_registered_lnn"] = str(CONFIG.lnn)
        kvs["esrs_registered_ip"] = str(CONFIG.ip)
        try:
            ns.update(kvs)
            break
        except tardis.TardisError as err:
            LOG.error(str(err))
            LOG.debug(traceback.format_exc())

    LOG.debug("finished update_tardis_registered_node")

def main():
    try:
        isi_daemon = daemon.Daemon(ISI_ESRS_DAEMON_NAME, daemonize=True)
    except daemon.DaemonException as daemon_err:
        exit(1)
    # We have to initialize this services logging after daemonizing or
    # for some reason, we end up closing the pidfile and release the
    # lock - messing up our ability to properly manage this service
    # see bug 212875
    init_logging()
    LOG.debug("isi_esrs_d started...")

    LOG.debug("isi_daemon initialized successfully")
    remove_esrs_eligible_flag()
    global CONFIG
    CONFIG = ESRSConfig()
    namespaces = [TARDIS_GCONFIG_ESRS_NS]
    filenames = [FLEXNET_CONFIG_XML_FILENAME]
    try:
        watchee_list = get_watchee_list(namespaces, filenames)
    except:
        remove_esrs_eligible_flag()
        exit(1)
    watcher = ESRSEventWatcher(watchee_list)

    while True:
        CONFIG.reload()
        if CONFIG.check_eligibility():
            # "thunderdome": if the node is currently eligible for the cluster
            # lock, attempt to take it and wait on it if unsuccessful
            LOG.info("Node {l} eligible for lock ownership"
                             .format(l=str(CONFIG.lnn)))
            LOG.info("Node {l} updating eligibility flag"
                             .format(l=str(CONFIG.lnn)))
            open(ESRS_ELIGIBLE_FLAG, "w").close()
            LOG.info("Node {l} waiting on lock"
                     .format(l=str(CONFIG.lnn)))
            isi_daemon.wait_master()
            LOG.info("Node {l} got lock"
                     .format(l=str(CONFIG.lnn)))
            break
        else:
            # wait on changes to ESRS or network config that may change
            # this node's eligibility
            LOG.info("Node {l} not currently eligible, waiting on change"
                     .format(l=str(CONFIG.lnn)))
            wait_on_kevent_watcher(watcher)

    CONFIG.reload()
    # if the node was eligible but was waiting a long time, it may have lost
    # eligibility while waiting, so it should recheck
    if not CONFIG.check_eligibility():
        remove_esrs_eligible_flag()
        LOG.info("Node {l} not currently eligible, exiting"
                 .format(l=str(CONFIG.lnn)))
        exit(0)

    # update the registered state tardis database with own lnn and desired
    # registered ip; this will also inform any watchers subscribed to
    # namespace changes that the lnn/ip were updated, and inform the ESRS
    # backend of the IP update (if ESRS is enabled)
    LOG.info("Node {l} updating registered node tardis variables"
             .format(l=str(CONFIG.lnn)))
    registered_node_ns = tardis.Namespace(TARDIS_REGISTERED_NODE_NS,
                                          tardis.TNS_O_CREAT)
    # attempt to update the registered IP on the ESRS backend; if ESRS is not
    # fully enabled yet, this transaction will not occur, but will be handled
    # by the RSAPI handler responsible for enabling
    esrs_ip_update()
    update_tardis_registered_node(registered_node_ns)

    LOG.debug("starting timer threads")
    # threading timer for usage intelligence calls
    ui_timer = Timer(CONFIG.ui_reporting_period, ui_thread)
    ui_timer.daemon = True
    ui_timer.start()
    # threading timer for ESRS get status requests
    status_timer = Timer(CONFIG.connectivity_check_period, status_thread)
    status_timer.daemon = True
    status_timer.start()
    # wait on changes to ESRS or network config that may change
    # eligibility; return if no long eligible
    LOG.info("Node {l} lock owner commencing waiting on change"
                     .format(l=str(CONFIG.lnn)))
    while True:
        # waiting on changes to gconfig tree via tardis subscription, flexnet
        # configuration; membership of majority group is automatically handled
        # by the isi_daemon library
        wait_on_kevent_watcher(watcher)
        # reload configuration after a change
        CONFIG.reload()

        # check if current node is still eligible
        if not CONFIG.check_eligibility():
            LOG.info("Node {l} lock owner no longer eligible, exiting..."
                     .format(l=str(CONFIG.lnn)))
            remove_esrs_eligible_flag()
            exit(0)

if __name__ == "__main__":
    sys.excepthook = log_exc

    # https://bugs.west.isilon.com/show_bug.cgi?id=215202
    # https://bugs.python.org/issue1230540
    init_old = threading.Thread.__init__

    def init(self, *args, **kwargs):
        init_old(self, *args, **kwargs)
        run_old = self.run

        def run_with_except_hook(*args, **kw):
            try:
                run_old(*args, **kw)
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception:
                sys.excepthook(*sys.exc_info())
        self.run = run_with_except_hook
    threading.Thread.__init__ = init

    main()
