#!/usr/libexec/isilon/isi_configwatch

# Copyright C 2002-2011 by Isilon Systems LLC. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
# 
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
# 
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems. 
#  
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.

# if success, return 0, other value means a Unexpected return

import os
import sys
import errno
import logging
import signal
import time
import string
import binascii
import threading
import subprocess
import getopt
import logging.handlers as handlers
import isi.config.arrcfg as ac
import isi.config.join as join
import isi.config.compliance as compliance
import isi.error as ierr
import isi.hw.hal as hal
import isi.hw.version as hwver
import isi.ui.lcd.screens as screens
import isi.ui.lcd.library as library
import isi.ui.lcd.config as config
import isi.ui.lib.format as fmt
import isi.ui.shell.cli_wizard as cliwiz
import isi.ui.shell.wizard as wiz
import isi.ui.shell.cli_console as console
import isi.ui.shell.CfgValidation as cfgv
import isi.ui.shell.AutoClusterSetup as acs

NEWFS_WAIT_FILE = '/etc/ifs/isi_newfs_wait'
IBOOTD_PID_FILE = '/var/run/isi_boot_d.pid'
ISI_RESET = "/usr/sbin/isi_reformat_node"
ISI_JOIN_D = "/usr/bin/isi_join_d"
IJOIND_PID_FILE = ''
# cluster guid file
GUID_FILE = ''
PLEASE_WAIT = 'Please wait...'
CANCEL = 'cancel'
AUTO_JOIN_RETRIES = 24
LCD_THREAD_JOIN_TIMEOUT = 5.0
KILL_LCD_D_TIMEOUT = 120

ignore_configured = False

log = logging.getLogger('isi_configwatch')

def initLogging():
    """Initializes logging systems with desired handler,
       formatter, and level. 
       """
    logs = {
        'isi_configwatch'   : logging.INFO,
    }

    # SysLogHandler with newsyslog tool will handle log rotation    
    handler = handlers.SysLogHandler(address='/var/run/log')
    # syslogd depends on a certain format - %(name)s has to be the first one 
    formatter = logging.Formatter('%(name)s[%(process)d]'': ' 
        '%(levelname)-8s %(message)s')
    handler.setFormatter(formatter)
      
    for (name, level) in logs.items():
        logger = logging.getLogger(name)
        logger.setLevel(level)
        logger.addHandler(handler)

    # Init the LCD library logging so our log message go into isi_lcd_d's
    # log file instead of ours. This makes debugging LCD timing issues
    # MUCH easier.
    library.init_client(logging.INFO)

class ScreenFlag:
    def __init__(self):
        self.stopFlag = False
        self._lock = threading.RLock()

    def set(self):
        self._lock.acquire()
        try:
            log.info("ScreenFlag: setting stopFlag")
            self.stopFlag = True
        finally:
            self._lock.release()

    def reset(self):
        self._lock.acquire()
        try:
            log.info("ScreenFlag: resetting stopFlag")
            self.stopFlag = False
        finally:
            self._lock.release()

    def stop(self):
        self._lock.acquire()
        try:
            ret = self.stopFlag
        finally:
            self._lock.release()
        return ret

class Status:
    def __init__(self):
        self._lock = threading.RLock()
        self._status = {
            'message': 'initializing isi_configwatch',
            'state': 0,
            'internal_error': False,
            'reboot_needed': False,
            'version_ok': False,
            'upgrade_needed': False,
        }

    def set_internal_error(self):
        self._lock.acquire()
        try:
            log.info("Status: setting internal error")
            self._status['internal_error'] = True
        finally:
            self._lock.release()

    def has_internal_error(self):
        ret = False
        self._lock.acquire()
        try:
            ret = self._status['internal_error']
        finally:
            self._lock.release()
        return ret

    def update(self, st):
        self._lock.acquire()
        try:
            log.info("Status: processing status update: "
                    "(%s, %X) -> (%s, %X)", self._status['message'],
                    self._status['state'], st.message, st.state)
            self._status['state'] = st.state
            self._status['message'] = st.message
            if st.message == 'Upgrade required':
                self._status['upgrade_needed'] = True
            if st.message == 'Versions compatible':
                self._status['version_ok'] = True
            if st.message == 'Preparation complete, reboot required':
                # NOTE: this status currently drives no action.
                self._status['reboot_needed'] = True
        finally:
            self._lock.release()

    def exiting(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['state'] == join.NODE_STATE_EXITING:
                ret = True
        finally:
            self._lock.release()
        return ret

    def expects_input(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['state'] == join.NODE_STATE_WAITING or \
                    self._status['state'] == join.NODE_STATE_ERROR:
                ret = True
        finally:
            self._lock.release()
        return ret

    # NOTE: currently this drives no action.  There is a desire to remove
    # the reboot from join_d and instead control it here.  This API would be
    # used to determine that reboot should be performed.
    def reboot_needed(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['reboot_needed']:
                ret = True
        finally:
            self._lock.release()
        return ret

    def upgrade_needed(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['upgrade_needed']:
                ret = True
        finally:
            self._lock.release()
        return ret

    def version_ok(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['version_ok']:
                ret = True
        finally:
            self._lock.release()
        return ret

    def working(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['state'] == join.NODE_STATE_WORKING:
                ret = True
        finally:
            self._lock.release()
        return ret

    def available(self):
        ret = False
        self._lock.acquire()
        try:
            if self._status['state'] == join.NODE_STATE_AVAILABLE:
                ret = True
        finally:
            self._lock.release()
        return ret

    def status_msg(self):
        ret = ""
        self._lock.acquire()
        try:
            ret = self._status['message']
        finally:
            self._lock.release()
        return ret

class Config:
    def __init__(self):
        self.clear()

    def clear(self):
        self._arrCfg = None
        self._flxCfg = None
        self._subnet = None
        self._pool = None
        # Flexnet can be: use internal as external OR configure its own ext-1
        self._flxmode = ''

# Global state of configwatch.
g_status = Status()
g_screenFlag = ScreenFlag()
g_cfgs = Config()

# Temporary solution
g_session = join.DiscoverySession()

# Pipe for StatusThread communicate with Wizard/Console CLI Thread
CliRd, StateWr1 = os.pipe()

# Handler which carries one-end of pipe to let thread get message from the
# other-end of pipe.
class MsgHdlr:
    def __init__(self, frd):
        self.rd = frd

    def process(self):
        return 'interrupt'

# Threads
class StatusThread(threading.Thread):
    def __init__(self, session, node, sflag):
        threading.Thread.__init__(self)
        self._session = session
        self._node = node
        self._screenFlag = sflag

    def run(self):
        global g_status
        global ignore_configured
        log.info("StatusThread: Starting.")
        st = self._node.status
        while True:
            try:
                if st.state == join.NODE_STATE_EXITING:
                    self.kill()
                    break
                st = self._session.wait_for_node_status_change(self._node, st)
                g_status.update(st)
                log.info("StatusThread: Status changes to: %s" %
                        g_status.status_msg())
                if ignore_configured and st.state == join.NODE_STATE_EXITING:
                    # disable debugging state
                    ignore_configured = False
                self.notify_all()
            except Exception, e:
                log.error("StatusThread: Error getting node status change: "
                        "%s" % str(e))
                g_status.set_internal_error()
                self.notify_all()
                self.kill()
                break

    def notify_all(self):
        os.write(StateWr1, "status changed")
        self._screenFlag.set()

    def kill(self):
        log.info("StatusThread: Terminating.")
        return

class MessageScreen(screens.OutputScreen):

    def __init__(self, text):
        screens.OutputScreen.__init__(self)
        self._text = text
        self._index = 0
        self._size = self.getDisplay().getNumRows()
        self._buffer = []

    def set_text(self, text, not_append=True):
        if (not_append):
            self._text = text
        else:
            self._text += text
        self.show()

    def show(self):
        if (not self.isActive()):
            self.activate()
        screens.Screen.show(self)
        width = self.getDisplay().NUM_COLS
        self._buffer = self.getDisplay().wrapText(self._text, width)
        self.show_message()

    def show_message(self):
        self.getDisplay().writeLines(
            [ self._buffer[i] for i in \
                  range(self._index, \
                            min(self._index + self._size, len(self._buffer))) ])

    def buttonUp(self):
        if len(self._buffer) > self._size:
            self._index = max(0, self._index - 1)
            self.show_message()

    def buttonDown(self):
        if len(self._buffer) > self._size:
            self._index = min(len(self._buffer) - self._size, self._index+1)
            self.show_message()

    def buttonCenter(self):
        self._state = self.STATE_EXITED

class NodeJoinInputScreen(screens.ListInputScreen):
    ERROR_MESSAGE = 'Unable to detect any configured clusters.'
    RESCAN = "rescan"

    def __init__(self, prompt, session):
        screens.ListInputScreen.__init__(self, prompt)
        self._session = session

    def activate(self):
        self.rescan()
        screens.ListInputScreen.activate(self)

    def rescan(self):
        clusters = self.listClusters()
        opts = [c.name for c in clusters]
        opts.append(self.RESCAN)
        self.setOptions(opts)

    def listClusters(self):
        clusterList = []
        try:
            clusterList = self._session.list_clusters()
        except:
            pass
        return clusterList

    def addNode(self):
        self.getDisplay().showMessage(PLEASE_WAIT)

        try:
            self._session.join_cluster_by_name(self.selection)
        except Exception, e:
            resp = "Join failed: %s" % str(e)
            self.getDisplay().showMessage(resp)
            return

        self.setOptions(None)
        self.commit()
        return

    def buttonCenter(self):
        if self.options and self.options[self.activeIdx] != CANCEL:
            if self.options[self.activeIdx] == self.RESCAN:
                self.rescan()
                self.changed = True
                self.show()
            else:
                self.selection = self.options[self.activeIdx]
                self.addNode()
                self.update()
        else:
            self._state = self.STATE_EXITED

    def commit(self):
        screens.ListInputScreen.commit(self)
        self.getDisplay().setTimeout(1)
        self._state = self.STATE_EXITED

class ErrorAckScreen(screens.ListInputScreen):
    def __init__(self, prompt, session, node):
        screens.InputScreen.__init__(self, prompt)
        self.setOptions(['confirm'])
        self._idx = 0
        self._lines = []
        self._size = self.getDisplay().getNumRows()
        self._session = session
        self._node = node

    def wrap_prompt(self):
        self._lines = self.getDisplay().wrapText(self.prompt,
                self.getDisplay().NUM_COLS)
        if len(self._lines[-1]) + self.inputWidth < \
                self.getDisplay().getNumCols():
            self.inputLoc = (len(self._lines[-1]) + 1, len(self._lines) - 1)
        else:
            self.inputLoc = (0, len(self._lines))
            self.inputWidth = self.getDisplay().getNumCols()

    def option_displayed(self):
        if self.inputLoc and self._idx + self._size <= self.inputLoc[1]:
            return False
        return True

    def show(self):
        screens.Screen.show(self)
        self.wrap_prompt()
        self.getDisplay().writeLines( \
            [ self._lines[i] for i in \
                  range(self._idx, \
                            min(self._idx + self._size, len(self._lines))) ])
        if not self.option_displayed():
            return
        if self.options:
            self.getDisplay().setTimeout(config.LCDConfig().getBlinkTime())
            self.update()

    def blink(self):
        if self.option_displayed() and self.inputLoc:
            # Write the current item to the screen (or blank for blinking)
            s = string.ljust("", self.inputWidth)
            if self.blinkOn and self.options:
                try:
                    s = self.options[self.activeIdx].ljust(self.inputWidth)
                except IndexError:
                    # If activeIdx is invalid, print a log message indicating
                    # what the current value is, and what the current options
                    # are, then go ahead and just display a blank string.
                    log.error("ErrorAckScreen.blink: activeIdx=%d is "
                              "invalid; options=%s" %
                              (self.activeIdx, self.options))
            self.getDisplay().showText(s, self.inputLoc[0], \
                                           self.inputLoc[1] - self._idx)

    def update(self):
        if not self.option_displayed():
            screens.Screen.update(self)
        else:
            screens.InputScreen.update(self)

    # The Up/Down button will scroll through the prompt and *then* 
    # select the options.
    def buttonUp(self):
        if not self.option_displayed() or self.activeIdx == 0:
            self._idx = max(0, self._idx - 1)
            self.show()
        else:
            screens.ListInputScreen.buttonUp(self)

    def buttonDown(self):
        if self.option_displayed():
            screens.ListInputScreen.buttonDown(self)
        else:
            self._idx = min(self.inputLoc[1] - self._size + 1, self._idx + 1)
            self.show()

    def buttonCenter(self):
        if not self.option_displayed():
            self._state = self.STATE_EXITED
            return

        if self.options and self.options[self.activeIdx] != CANCEL:
            self.selection = self.options[self.activeIdx]
            if self.selection == 'confirm':
                self._session.confirm_node_status(self._node.name)
            self._state = self.STATE_EXITED

class LcdThread(threading.Thread):
    def __init__(self, session, node, flag):
        threading.Thread.__init__(self)
        self._flag = flag
        self._session = session
        self._node = node

    def handle_expects_input(self):
        global g_status
        strMsg = "Joining status: %s" % g_status.status_msg()
        screen = ErrorAckScreen(strMsg, self._session, self._node)
        screen.CAN_CANCEL = False
        screen.CAN_EXIT = True
        log.info("pushing ErrorAckScreen to LCD")
        lcd = library.LCDClient(screen)
        lcd.start()
        while screen.isActive() and not self._flag.stop():
            lcd.service()
            try:
                screen.update()
            except EnvironmentError:
                pass
            lcd.service()
        lcd.cancel()

    def handle_join(self):
        screen = NodeJoinInputScreen("Select cluster to join:",
                self._session)
        screen.CAN_CANCEL = True
        screen.CAN_EXIT = False
        log.info("pushing NodeJoinInputScreen to LCD")
        lcd = library.LCDClient(screen)
        lcd.start()
        log.info("NodeJoinInputScreen got sid %d" % lcd._sid)
        while screen.isActive() and not self._flag.stop():
            lcd.service()
            try:
                screen.update()
            except EnvironmentError:
                pass
            lcd.service()
        lcd.cancel()

    def handle_status(self):
        global g_status
        strMsg = "Joining status: %s" % g_status.status_msg()
        screen = MessageScreen(strMsg)
        screen.CAN_CANCEL = False
        screen.CAN_EXIT = False
        log.info("pushing MessageScreen to LCD")
        lcd = library.LCDClient(screen)
        lcd.start()
        while screen.isActive() and not self._flag.stop():
            lcd.service()
            try:
                screen.update()
            except EnvironmentError:
                pass
            lcd.service()
        lcd.cancel()

    def run(self):
        global g_status
        log.info("LCDThread: Starting.")
        try:
            while True:
                if g_status.has_internal_error():
                    log.info("LCDThread: Status Internal Error")
                    self.kill()
                    break
                elif g_status.available():
                    log.info("LCDThread: Join")
                    self.handle_join()
                elif g_status.working():
                    log.info("LCDThread: Status")
                    self.handle_status()
                elif g_status.expects_input():
                    log.info("LCDThread: Expects Input")
                    self.handle_expects_input()
                elif g_status.exiting():
                    log.info("LCDThread: Exiting")
                    self.kill()
                    break
                if self._flag.stop():
                    log.info("LCDThread: resetting stop flag")
                    self._flag.reset()
        except Exception, e:
            log.error("Error connecting to LCD: %s" % str(e))

    def kill(self):
        log.info("LCDThread: Terminating")
        return

def has_guid_file():
    if os.path.exists(GUID_FILE):
        return True
    else:
        return False

# True if the file exist & we can parse it.
# TODO: use CfgValidation's validity_check method instead.
def valid_config():
    if ignore_configured:
        return False

    # check whether the array.xml exist
    if not os.path.exists("/etc/ifs/array.xml"):
        return False
    # if error_out equal NULL, means exist and valid
    try:
        cfg = ac.config_load()
    except ac.ConfigError:
        msg = "Error: Invalid cluster configuration detected in array.xml. " \
              "Halting boot process."
        log.error(msg)
        print >> sys.stderr, fmt.fmtUserMsg(msg)
        sys.exit(-1)

    if cfg.is_new():
        return False

    if cfg.find_device_by_devid(cfg.local_devid) is not None:
        return True

    return False

# Node is no longer member of cluster. 
class NoLongerMCmd(wiz.WizMenuCmd):
    def __init__(self, title, hdlr = None):
        wiz.WizMenuCmd.__init__(self, title, hdlr)

    def display(self):
        print "This node used to be part of a cluster but " \
            "the cluster no longer "
        print "recognizes this node and will not " \
            "allow it to rejoin the cluster."
        print "The only remedy available from this node is to " \
            "reset the node which will"
        print "destroy any data currently stored on the node."
        print
        print "Type 'yes' to reset and reformat this node."
        print "Type 'no' to continue booting and attempt " \
            "offline recovery."

    def do_yes(self, args):
        print "Resetting and reformatting node to clean state."
        # Passing second param to preserve logs and cores.
        buf = "%s yes -P > /dev/null 2>&1" % ISI_RESET
        subprocess.call(buf, shell=True, close_fds=True)
        #Sleep so that we don't do anything else while waiting for reboot
        time.sleep(600)

    def do_no(self, args):
        print "Continuing boot without reset/reformat."
        sys.exit(0)

    def default(self, args):
        print "Please enter 'yes' or 'no'"

class ExpectsInputMenuCmd(wiz.WizMenuCmd):
    def __init__(self, title, hdlr = None, session = None):
        wiz.WizMenuCmd.__init__(self, title, hdlr)
        self._session = session

    def display(self):
        global g_status
        print >> sys.stderr, fmt.fmtUserMsg("Error: %s" %
                g_status.status_msg())
        print "Hit [Enter] to acknowledge error and return to join menu."

    def _confirm(self):
        try:
            self._session.confirm_local_node_status()
        except Exception, e:
            print >> sys.stderr, fmt.fmtUserMsg(
                    "Error confirming node status: %s" % str(e))
            return 'failed'
        return 'succeeded'

    def do_yes(self, args):
        self._ret_cmd = self._confirm()
        return -1

    def default(self, args):
        pass

    def emptyline(self):
        self._ret_cmd = self._confirm()
        return -1

def check_membership():
    assert valid_config(), "valid config not found!"

    cfg = ac.config_load()
    dev = cfg.find_device_by_guid(ac.get_local_guid())
    if dev and ac.get_local_devid() == 0:
        # Somehow our local devid (in local.xml) is not set but we
        # belong to a cluster, so re-store our config and it will
        # automatically put our devid into local.xml for us.
        cfg.store()

    if dev:
        return True
    else:
        return False

def start_node_join_daemon(update_status=False):
    try:
        # First try talking to it to see if it's there
        node = g_session.get_local_node()
        if update_status:
            log.info("Updating global status: %s -> %s" %
                (g_status.status_msg(), node.status))
            g_status.update(node.status)
        else:
            log.info("Current global status: %s" % g_status.status_msg())
        return node
    except Exception, e:
        # If we can't talk to it, see if there's a non-responsive one out
        # there; kill anything non-responsive
        pid = acs.get_pid("/var/run/isi_join_d.pid")
        if pid > 0:
            log.error("Error contacting node-join daemon: %s" % str(e))
            os.kill(pid, signal.SIGKILL) #forcibly kill
            time.sleep(1)

        log.info("Starting node-join daemon")
        subprocess.call(ISI_JOIN_D + " -n", shell=True, close_fds=True)
        timeout = 5
        while timeout > 0:
            try:
                # If we've re/started isi_join_d, always update the global
                # status object.
                node = g_session.get_local_node()
                g_status.update(node.status)
                return node
            except ValueError:
                timeout = timeout - 1
                time.sleep(1)
            except Exception, e:
                msg = "Error getting local node: %s" % str(e)
                log.error(msg)
                print >> sys.stderr, fmt.fmtUserMsg(msg)
                sys.exit(1)

        if timeout <= 0:
            msg = "Error: Failed to start node-join daemon on local node."
            log.error(msg)
            print >> sys.stderr, fmt.fmtUserMsg(msg)
            sys.exit(1)

def parse_options(args):
    try:
        (opts, params) = getopt.gnu_getopt(args[1:], '',
                                           ['ignore-configured'])
    except getopt.GetoptError, e:
        opts = []
        print >> sys.stderr, fmt.fmtUserMsg('Error: %s' % e.msg)
    # Handle options
    for (opt, optarg) in opts:
        if opt == '--ignore-configured':
            global ignore_configured
            ignore_configured = True

# If NEWFS_WAIT_FILE is set, newfs_efs was not run during isi_first_boot.
# Instead, we wait to run newfs_efs when cluster create is complete or
# on a node join.
def newfs_efs():
    result = 0
    if os.path.exists(NEWFS_WAIT_FILE):
        # If we need to run newfs, isi_km_d must be available
        # to ensure proper access to SED drives.  isi_km_d has a
        # dependency on rpcbind which has a dependency on isi_ipmi_d.
        # Start them all in the correct order.
        os.system('/usr/sbin/isi_ipmi_d')
        os.system('/usr/sbin/rpcbind')
        os.system('/usr/libexec/isilon/isi_km_d')
        os.system('/sbin/newfs_efs -f')
        os.unlink(NEWFS_WAIT_FILE)
        result = subprocess.call('/usr/sbin/isi_testjournal', shell=True)
        if result:
            msg = 'Test journal exited with an error.'
            print >> sys.stderr, fmt.fmtUserMsg(msg)
    return result

def main():
    global g_status, g_session, g_cfgs
    node = None
    thread_flag = False
    initLogging()
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    lcdThread = None

    parse_options(sys.argv)

    compliance.set_compliance_sysctl()

    # If we have a valid array.xml, upgrade it if necessary and make sure
    # this local node has a devid. The array.xml upgrade has to happen as
    # early as possible since so many things try to read it and we want to
    # be able to stop boot if something is amiss.
    if valid_config():
        try:
            if not ac.is_current():
                ac.config_upgrade()
        except ac.Error, e:
            msg = "Error validating array.xml: %s" % str(e)
            log.error(msg)
            print >> sys.stderr, fmt.fmtUserMsg(msg)
            sys.exit(1)

        if not check_membership():
            nlm = NoLongerMCmd('')
            nlm.cmdloop()

    if not valid_config():
        # We want to update the global status the first time through.
        node = start_node_join_daemon(update_status=True)

        # Start the Status Thread to monitor join_d status change.
        statusThread = StatusThread(g_session, node, g_screenFlag)
        statusThread.start()

        # Start papi daemon for configuration interface
        acs.restart_papi_daemon(unconfigured=True)

        # Start the LCD Thread if we have LCD screen.
        if hal.getLcdType() != hwver.LCDVER_UNKNOWN:
            lcdThread = LcdThread(g_session, node, g_screenFlag)
            lcdThread.start()

        thread_flag = True

        if os.path.exists(join.CLUSTER_TO_JOIN_FILE):
            guid_file = open(join.CLUSTER_TO_JOIN_FILE)
            cluster_guid = guid_file.read()
            readable_guid = binascii.hexlify(cluster_guid)
            os.unlink(join.CLUSTER_TO_JOIN_FILE)

            ds = join.DiscoverySession()
            retries = 0
            while retries < AUTO_JOIN_RETRIES:
                try:
                    ds.join_cluster_by_guid(cluster_guid,
                            upgrade_if_needed="no",
                            options_from_file=join.OPTIONS_TO_JOIN_FILE)
                except (ierr.IsiError, ValueError), e:
                    retries += 1
                    msg = "Error: Failed to automatically join cluster " \
                            "%s after re-versioning: %s" % \
                            (readable_guid, str(e))
                    if retries < AUTO_JOIN_RETRIES:
                        msg += " - retrying"
                    else:
                        print >> sys.stderr, fmt.fmtUserMsg(msg)

                    log.info(msg)
                    continue

                # We successfully sent our join request to the cluster,
                # give isi_join_d some time to do its thing and then break
                # out of loop.
                # 
                # However, if we support self encrypting drives, we need to
                # newfs the drives now before we move on.
                if hal.supportsSelfEncryptingDrives():
                    if newfs_efs():
                        sys.exit(1)
                else:
                    time.sleep(5)
                break

    if os.path.exists(join.OPTIONS_TO_JOIN_FILE):
        os.unlink(join.OPTIONS_TO_JOIN_FILE)

    wizMsgHdlr = MsgHdlr(CliRd)
    next = "wizard"
    while not valid_config():
        # Make sure join_d is running just in case it died!
        node = start_node_join_daemon(update_status=False)
        # Start the Status Thread and LCD Thread again if necessary
        if not statusThread.isAlive():
            statusThread = StatusThread(g_session, node, g_screenFlag)
            statusThread.start()

        if hal.getLcdType() != hwver.LCDVER_UNKNOWN:
            if lcdThread is None or not lcdThread.isAlive():
                lcdThread = LcdThread(g_session, node, g_screenFlag)
                lcdThread.start()

        if g_cfgs._arrCfg is None:
            g_cfgs._arrCfg = ac.config_load()

        if not g_status.available():
            next = "interrupt"

        if next == "wizard":
            signal.signal(signal.SIGINT, signal.SIG_DFL)
            # Let wizard to capture the signal.
            next = cliwiz.run_wizard(g_cfgs, wizMsgHdlr)
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        if next == "manual":
            next = console.run_console(g_cfgs, wizMsgHdlr)

        if next == "exit":
            g_cfgs.clear()
            next = "wizard"

        if next == "commit":
            g_cfgs.clear()

        if next not in ["hold", "manual", "wizard", "commit"]:
            # g_cfgs variable cleanup.
            if g_cfgs._flxCfg is not None:
                g_cfgs._flxCfg.close()
            g_cfgs.clear()

        if next == "hold":
            next = "wizard"

        # User is creating a new cluster
        if next == "commit":
            if hal.supportsSelfEncryptingDrives():
                if newfs_efs():
                    sys.exit(1)

        if next in ['interrupt', 'joined']:
            if hal.supportsSelfEncryptingDrives():
                # Block while join_d determines if an upgrade is needed
                while (not g_status.version_ok() and
                    not g_status.upgrade_needed()):
                    time.sleep(5)
                # Version is the same as the cluster - no reimage needed
                if g_status.version_ok():
                    if newfs_efs():
                        sys.exit(1)

        if next == "commit":
            try:
                g_session.shutdown_local_node()
            except ValueError:
                # We may have just been configured by someone else (bug 50514)
                # restart!
                print "Failed to commit to local node.  Configured remotely?"
                continue

        if next in ['interrupt', 'joined']:
            restartWizard = False
            while True:
                if g_status.has_internal_error():
                    # all we can do is restart the wizard!
                    # but we need to restart the status thread too!
                    log.error("MainThread: Encounter an internal error.")
                    next = "wizard"
                    break

                if g_status.working():
                    print "Joining status: %s" % g_status.status_msg()
                    # NOTE: falls through to the blocking read below

                if g_status.available():
                    if restartWizard or next == "interrupt":
                        next = "wizard"
                        break

                if g_status.exiting():
                    print "Joining status: %s" % g_status.status_msg()
                    break

                if g_status.expects_input():
                    expectsInputMenuCmd = ExpectsInputMenuCmd("", wizMsgHdlr,
                                                              g_session)
                    next = expectsInputMenuCmd.cmdloop()
                    log.info("User Input confirm prompt returned %s" % next)
                    restartWizard = True
                    # If we were interrupted, status probably changed so
                    # repeat this inner loop to take appropriate action.
                    if next in ['interrupt']:
                        continue

                    # If the confirm failed or the user wanted out, bail
                    # out and restart the wizard.
                    if next in ['quit', 'back', 'failed']:
                        # Normalize to what main loop expects
                        next = 'exit'
                        break

                    # NOTE: next == 'succeeded' falls through to the
                    # blocking read below.

                os.read(CliRd, 8192)

    assert valid_config(), \
            "MainThread: main loop exited without a valid config!"

    # Killing papi unconfigured mode
    acs.kill_daemon(acs.IPAPID_PID_FILE)

    log.info("MainThread: Starting isi_boot_d")
    subprocess.call("/usr/sbin/isi_boot_ctl start", shell=True,
            close_fds=True)

    # We have to restart syslogd after we bring up the backend network,
    # which isi_boot_d (started above) handles.

    log.info("MainThread: Restarting syslogd")
    subprocess.call("/usr/bin/isi_lcd_d_stop", shell=True,
            close_fds=True)
    # XXX:HACK
    end_time = time.time() + KILL_LCD_D_TIMEOUT
    while not subprocess.call("/usr/bin/killall -0 isi_lcd_d", shell=True):
        time.sleep(5.0)
        if time.time() > end_time:
            # we have exceeded the timeout window, send a -9
            # see bug 131820
            subprocess.call("/usr/bin/killall -9 isi_lcd_d", shell=True)
            break
    subprocess.call("/etc/rc.d/syslogd stop", shell=True,
            close_fds=True)
    subprocess.call("/etc/rc.d/syslogd start", shell=True,
            close_fds=True)
    subprocess.call("/usr/bin/isi_lcd_d start", shell=True,
            close_fds=True)

    # XXXSVW: Temporary debug logging for bug 60713. Log ps output
    # grep'd for isi_lcd_d processes.
    pout = subprocess.Popen("/bin/ps -auwwx | /usr/bin/grep isi_lcd_d",
            shell=True, bufsize=1024, stdout=subprocess.PIPE).stdout
    log.info("MainThread: LCD process information:")
    for l in pout.readlines():
        log.info("MainThread:... %s", l)

    # All status need to join the main thread.
    if thread_flag:
        log.info("MainThread: Waiting for StatusThread")
        statusThread.join()
        log.info("MainThread: StatusThread exited")

        if lcdThread is not None:
            log.info("MainThread: Waiting for LcdThread")

            # This is a work-around for bug 60713, a bug somewhere in the
            # LCD that causes the LCDClient to hang, which in turn causes
            # our LcdThread to never exit/join. At this point in a join,
            # we've already received the array.xml and started up
            # isi_boot_d, so there's really no reason to let this
            # disrupt the join process.
            lcdThread.join(LCD_THREAD_JOIN_TIMEOUT)
            if lcdThread.isAlive():
                log.info("MainThread: LcdThread didn't exit after %.1f "
                        "seconds; giving up waiting",
                        LCD_THREAD_JOIN_TIMEOUT)
            else:
                log.info("MainThread: LcdThread exited")

    signal.signal(signal.SIGINT, signal.SIG_DFL)

    log.info("MainThread: Exiting")
    sys.exit(0)

if __name__ == '__main__':
    main()

# vim:set filetype=python:
