#!/usr/libexec/isilon/isi_smartpools_d

# isi_smartpools_d
# =====================
#
# Keeps the diskpool db in sync with the drivepurpose db
#

import os
import os.path
import select
from signal import SIGHUP, SIGPIPE, SIGALRM, SIGWINCH, SIGINFO, SIGURG, \
        SIGIO, SIGUSR1, SIGUSR2, SIGINT, SIGTERM, SIGQUORUM, SIGKILL
import socket
import sys
import syslog
from syslog import LOG_ERR, LOG_INFO, LOG_CRIT, LOG_PERROR, LOG_NOWAIT, \
        LOG_USER, LOG_PID, LOG_NDELAY, LOG_DEBUG, LOG_WARNING, LOG_NOTICE
import time
import traceback
import errno
import isi.app.daemon
import isi.misc
import isi.fs.procoptions
import isi.fs.diskpool as fsdiskpool
import isi.fs.drive_purposing as fsdp
import isi.smartpools.diskpools as spdp
import isi.smartpools.filepools as spfp

IfsVarDirectory = "/ifs/.ifsvar"
_StateDirectory = "modules/smartpools_d"
StateDirectory = IfsVarDirectory + "/" + _StateDirectory
LeaderFile = StateDirectory + "/leader.pid.lk"
MinStateVersion = 1
StateVersion = 1

Debug = False

# Every 60 minutes, try to conform anyways
EVENT_WAIT_TIMEOUT = 60 * 60

KQ_VNODE_ALL = select.KQ_NOTE_DELETE | select.KQ_NOTE_WRITE | \
        select.KQ_NOTE_EXTEND | select.KQ_NOTE_LINK | \
        select.KQ_NOTE_RENAME | select.KQ_NOTE_REVOKE

def usage():
    print "isi_smartpools_d: Smartpools syncronization daemon"
    print ""
    print "  Usage: configure in SmartPools WebUI"
    print "    -D: Debug: run in foreground"


def main():
    global Debug
    argv = sys.argv[1:]
    while argv:
        arg = argv.pop(0)
        if arg == "-D":
            Debug = True
            continue

        usage()
        sys.exit(1)

    # MCP doesn't tolerate daemons staying in the foreground. So, we must
    # background ourselves.

    # Backgrounds, locks pidfile /var/run/isi_smartpools_d.pid, and writes
    # our pid out Note: Backgrounding must happen before pretty much everything
    # else.
    if not Debug:
        isi.app.daemon.daemonize()

    os.umask(0)

    syslog.openlog("isi_smartpools_d", LOG_PID | LOG_NDELAY | LOG_PERROR,
            LOG_USER)

    syslog.syslog(LOG_INFO, "Daemon starting")
    run_as_daemon()


def run_as_daemon():
    isi.misc.wait_on_quorum()
    erofs = False

    while True:
        try:
            run_uncaught()
            erofs = False
        except EnvironmentError as ee:
            if ee.errno != errno.EROFS:
                raise

            if not erofs:
                syslog.syslog(LOG_ERR,
                        "Filesystem is read-only, can't procede")
                # Only log once per outage.
                erofs = True

            time.sleep(30)

def run_uncaught():
    if isi.fs.procoptions.getprocenc() != isi.fs.procoptions.ENC_UTF8:
        syslog.syslog(LOG_INFO, "Setting per-process encoding to UTF-8")
        isi.fs.procoptions.setprocoptions(enc=isi.fs.procoptions.ENC_UTF8)

    create_state_directory()

    kq = select.kqueue()
    dpfd = -1

    kq.control([make_dp_kev([dpfd])], 0, 0)
    kq.control([select.kevent(SIGHUP, filter=select.KQ_FILTER_SIGNAL)], 0, 0)

    while True:
        if i_am_leader():
            conform_diskpool_db_to_drive_purpose()

        evs = kq.control(None, 5, EVENT_WAIT_TIMEOUT)

        # We do not care what the event(s) were, but we need to reseat the
        # vnode event. Re-scan (leader) or attempt coup (gremlins).
        for ev in evs:
            if Debug:
                syslog.syslog(LOG_CRIT, "XXX kevent(): Got event %r" % ev)

            if ev.filter == select.KQ_FILTER_VNODE:
                # Delete the event and try to re-seat it on the more specific
                # file fd. (Also, the database may have deleted the old vnode,
                # so we need to find the new vnode to watch.)
                ev.fflags = KQ_VNODE_ALL
                ev.flags = select.KQ_EV_DELETE
                kq.control([ev], 0, 0)
                os.close(ev.ident)
                kq.control([make_dp_kev([dpfd])], 0, 0)

    sys.exit(0)

def conform_diskpool_db_to_drive_purpose():
    # Make sure we're upgraded and needs a conform
    dpdb = isi.fs.drive_purposing.DrivePurposeDB()
    if not dpdb.fully_upgraded() or not dpdb.needs_conform():
        return

    syslog.syslog(LOG_INFO, "Conforming diskpool db to drive_purpose_db")

    MAX_NUM_ATTEMPTS = 10
    RETRY_DELAY = 1
    attempts = 0

    # Conform and try to write the diskpool db
    while attempts < MAX_NUM_ATTEMPTS:
        # open the filepool policy config
        dp_cfg = spdp.open_config()
        fp_cfg = spfp.open_config(dp_cfg._ctx._gcfg_ctx)
        assert fp_cfg != None # Assert to help find Bug 129297
        # Should not run when the diskpool db is not upgraded
        if not dp_cfg.db.fully_upgraded:
            return

        # conform nodepools to match the drive_purpose_db
        needs_write = dp_cfg.conform_provisioning_to_node_types(fp_cfg)
        if not needs_write:
            syslog.syslog(LOG_INFO, "No write necessary")
            break

        # Must write the fp_policies first
        syslog.syslog(LOG_INFO, "Writing the fp policy db")
        try:
            fp_cfg.save(commit_ctx=False)
        except Exception as e:
            syslog.syslog(LOG_CRIT, "XXX Could not write updated filepool "
                    "policies with exception: %s. Proceeding anyway." % str(e))

        syslog.syslog(LOG_INFO, "Writing the diskpool db")
        try:
            dp_cfg.save(ignore_underprovisioned=False)
            break
        except OSError, e:
            if e.errno in [errno.ESTALE, errno.EAGAIN, errno.ENOENT]:
                # Sleep for a couple seconds before trying again.
                syslog.syslog(LOG_CRIT, "XXX Failed to write diskpool db, "
                        "trying again.")

                time.sleep(RETRY_DELAY)
                attempts += 1
                # and enable debugging to /var/log/diskpools.log
                import isi.fs.diskpool as fsdp
                fsdp.set_debug(True)
                continue
            else:
                raise

    if attempts >= MAX_NUM_ATTEMPTS:
        syslog.syslog(LOG_CRIT, "XXX Could not conform diskpool db to"
                "drive_purpose_db")
        return

    # Fix the suggested protection of the new eqs and unset the needs_conform
    # flag
    attempts = 0
    while attempts < MAX_NUM_ATTEMPTS:
        dpdb = isi.fs.drive_purposing.DrivePurposeDB()

        # Update each suggested protection that is None:
        syslog.syslog(LOG_INFO, "Updating suggested protections")
        for eq_entry in dpdb.eq_entries().values():
            if not eq_entry.get("suggested_protection", None):
                dpdb.update_suggested_protection(eq_entry)

        dpdb.set_needs_conform(False)
        try:
            syslog.syslog(LOG_INFO, "Unsetting needs_conform flag")
            dpdb.write()
        except OSError, e:
            if e.errno in [errno.ESTALE, errno.EAGAIN, errno.ENOENT]:
                # Sleep for a couple seconds before trying again.
                syslog.syslog(LOG_CRIT, "XXX Failed to write drive_purpose_db,"
                        " trying again.")
                time.sleep(RETRY_DELAY)
                attempts += 1
                continue
            else:
                raise

        return

    if attempts == MAX_NUM_ATTEMPTS:
        syslog.syslog(LOG_CRIT, "XXX Could not unset the needs conform flag in "
                "the drive_purpose_db")
        return

def log_exc(e):
    tb = traceback.format_exc()
    syslog.syslog(LOG_WARNING, "Exception: %s: %s" % (str(e), tb))
    log = "%s: %s" % (str(e), tb)
    for line in log.strip().split("\n"):
        syslog.syslog(LOG_INFO, line)

def create_state_directory():
    if os.path.exists(StateDirectory):
        return

    if not os.path.exists(IfsVarDirectory):
        die(".ifsvar does not exist -- maybe /ifs isn't mounted?")

    try:
        os.makedirs(StateDirectory, 0755)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

def make_dp_kev(dpfdp):
    # Watch the parent directory of the drive purpose db (which will get
    # kevents on the db iff the db is written using the supported pattern of
    # write to tmp file, and rename it to the db name)
    dpfdp[0] = os.open(os.path.dirname(fsdp.DB_FILE), os.O_RDONLY)

    return select.kevent(dpfdp[0], filter=select.KQ_FILTER_VNODE,
            fflags=KQ_VNODE_ALL)


LeaderPidfile = None
IAmLeader = False
def i_am_leader():  # -> bool
    global LeaderPidfile, IAmLeader

    if not IAmLeader:
        psuedo_pid = "devid:%d, pid:%d" % (isi.config.arrcfg.get_local_devid(),
                os.getpid())

        LeaderPidfile = isi.app.daemon.PidFile(filename=LeaderFile)
        try:
            LeaderPidfile.write(psuedo_pid, die_on_error=False)
            IAmLeader = True
        except IOError as ioe:
            if ioe.errno not in (errno.EACCES, errno.EAGAIN):
                raise

    return IAmLeader

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        log_exc(e)
        raise
