#!/usr/bin/python
# ex: set shiftwidth=4 tabstop=4 expandtab noai indentexpr= textwidth=78:
# $Id$
""" isi_tools -i [-fn] [--latest|--build <build>] [--target <dir>]
 isi_tools -g
 isi_tools -s <0/1>

  -i, --install         Install the engineering tools package for this build.
  -g, --get-internal    Print whether this node is an internal Isilon node.
  -s, --set-internal    Change whether this node is an Isilon internal node.
                        [Note: This will pull and push from the dongle]

 Installation Options:
  -b, --build <build>   Install the tools package for a specific build
  -f, --fresh           Skip install if there is an existing installation.
  -l, --latest          Install the latest tools package built on this branch.
  -n, --internal-only   Only install if the node is an Isilon internal node.
  -t, --target <dir>    Install the package under the specified directory.
                        [default: /var/crash]
  -w, --wait            Wait for network to become available before install.

 General Options:
  -q, --quiet           Reduces output; may be specified twice for no output.
"""

import errno
import getopt
import os
import re
import shutil
import stat
import sys
import time
import urllib2

try:
    import isi.net.interface as ni
    pre_flexnet_merge = True
except:
    import isi.net.flexnet as flx
    import isi.config.ifs as ifs
    pre_flexnet_merge = False

import isi.app.lib as applib

from commands import getstatusoutput as run
import isi.hw.hal as hal

# globals
hwversions_path  = "/etc/isilon_hwversions"
installation_dir = "/var/crash"        # default installation target
tarfile          = "/var/crash/engtools.tar"
engtools_lock    = "/var/crash/engtools.lock"

# these are used to categorize output levels; see log() and -q
output_level     = 2    # this level of messages and below will
                        # be displayed.
output_error     = 1
output_log       = 2

# If this file exists, it means the tools package has been installed.
installed_file   = "/etc/isilon_tools_installed"


class Usage(Exception):
    """Exception raised when the program is being used incorrectly"""
    def __init__(self, msg):
        self.msg = msg

class Failure(Exception):
    """Exception used for all other failures"""
    def __init__(self, msg):
        self.msg = msg

def get_build_info(build=None):
    """Determines the running build and returns a hash with information
pertaining to that build."""
    if build is None:
        version = os.uname()[3]
        try:
            build = re.findall(" (B_\S*)", version)[0]
        except:
            raise Failure("failed to parse build: %s" % version)

    # normalize build to lower case with .'s
    normalized_build = re.sub(r"_", ".", build.lower())

    # matches B_1234(RELEASE)
    match = re.match(r"^b\.?(.*)\.(\d+)\((.*)\)$", normalized_build)
    if match is None:
        # matches b.1234r
        match = re.match(r"^b\.?(.*)\.(\d+)(.)$", normalized_build)
    try:
        (branch, build_number, build_type) = match.groups()
    except AttributeError, e:
        raise Failure("could not parse build: %s" % build)

    if build_type in ("r", "release"):
        build_type = "release"
    else:       # for DEBUG and things like WITH_INVARIANTS
        build_type = "debug"

    # format the build as used in snapshots dir on buildbiox
    if branch:
        snapshots_path = "b.%s.%s%c" % (branch, build_number, build_type[0])
        fmt_branch = re.sub(r"\.", "_", branch)
        latest_snapshots_path = "latest.BR_%s/obj.%s" % (fmt_branch.upper(),
                                                         build_type.upper())
    else:
        snapshots_path ="b.%s%c" % (build_number, build_type[0])
        latest_snapshots_path = "latest.HEAD/obj.%s" % (build_type.upper())

    build_info = {
        "build": normalized_build,
        "branch": branch,
        "build_number": build_number,
        "build_type": build_type,
        "latest_snapshots_path": latest_snapshots_path,
        "snapshots_path": snapshots_path
    }
    return build_info

def get_tools_tar(latest=False, save_as=None, build_info=None):
    """retrieves the tools tar and returns the path to it on disk.
note that the tar will be deleted when the program exists"""
    if build_info is None:
        build_info = get_build_info()

    if save_as is None:
        save_as = tarfile

    try:
        os.unlink(save_as)
    except OSError, e:
        if e.errno == errno.ENOENT:
            pass
        else:
            raise

    url = "http://buildbiox.west.isilon.com/snapshots/"
    if latest:
        url += "%s/engtools.tar" % ( build_info["latest_snapshots_path"], )
    else:
        url += "%s/obj/engtools.tar" % ( build_info["snapshots_path"], )

    # add support for compressed tools tars
    save_as_gz = save_as + ".gz"
    url_gz = url + ".gz"

    # try to retrieve the gzipped tarball, but fallback to the
    # uncompressed version if the compressed is not available.
    log("Retrieving %s" % url_gz)
    filename = get_to_file(url_gz, save_as_gz, notfound_ok=True)
    if filename is None:
        log("Retrieving %s" % url)
        filename = get_to_file(url, save_as)

    return filename

def get_to_file(url, filename, notfound_ok=False):
    """wrapper to use urllib2 to retrieve a webpage to a file"""
    try:
        error = None
        blocksize = 32768
        u = urllib2.urlopen(url)

        f = file(filename, "w")
        data = u.read(blocksize)
        while data != '':
            f.write(data)
            data = u.read(blocksize)
        f.close()
        return filename
    except urllib2.HTTPError, e:
        if notfound_ok and e.code == 404:
            filename = None
        else:
            error = e
    except urllib2.URLError, e:
        error = e.reason[1]

    if error is not None:
        raise Failure("Error retrieving tools tar: %s" % error)

    return filename

def install_tools_tar(filename, target):
    """installs the tarball specified into the "tools" subdirectory of target
NOTE: this will delete any existing installation in that directory"""
    target = "%s/tools" % (target, )

    # clean up the directory if it exists
    try:
        shutil.rmtree(target)
    except OSError, e:
        if e.errno == errno.ENOENT:
            pass
        else:
            raise

    # create the directory and install the files
    log("Extracting tools package to %s" % target)
    os.makedirs(target)

    if filename.endswith(".gz"):
        tar_cmd = "tar -C %s -zxf %s"
    else:
        tar_cmd = "tar -C %s -xf %s"
    (status, output) = run(tar_cmd % (target, filename))
    if status & 127:
        raise Failure("tar exited with signal %d" % (status & 127))
    elif status != 0:
        raise Failure("tar returned %d" % (status >> 8))

    # run the installer from the tools tar, if it exists.
    try:
        installer = target + "/etc/installer"
        if os.stat(installer).st_mode & stat.S_IEXEC:
            log("Running installer...")
            (status, output) = run("%s %s" % (installer, target))
            if output:
                log("installer output:\n\n%s\n" % output)
            if status & 127:
                raise Failure("installer exited with signal %d" % \
                        (status & 127))
            elif status != 0:
                raise Failure("installer returned %d" % (status >> 8))
    except OSError, e:
        if e.errno == errno.ENOENT:
            pass
        else:
            raise

    # TODO installer code should run here if an installation script is
    # ever included
    try:
        nose_path = target + "/tmp/nose"
        nose_installer = nose_path + "/setup.py"
        if os.path.exists(nose_installer):
            log("Installing nose...")
            (status, output) = run("cd %s && /usr/bin/python %s install" % \
                        (nose_path, nose_installer))
            if output:
                log("installer output:\n\n%s\n" % output)
            if status & 127:
                raise Failure("nose install exited with signal %d" % \
                        (status & 127))
            elif status != 0:
                raise Failure("nose install returned %d" % (status >> 8))
            log("Removing temp directory...")
            shutil.rmtree(nose_path)
    except OSError, e:
        if e.errno == errno.ENOENT:
            pass
        else:
            raise

    # isi_tools.pth gets added to tarball as part of build.
    # Copy isi_tools.pth file into the python include path
    try:
        from distutils.sysconfig import get_python_lib
        pth_src = target + "/etc/isi_tools.pth"
        pth_dest = get_python_lib() + '/isi_tools.pth'
        if os.path.exists(pth_src):
            shutil.copyfile(pth_src, pth_dest)
    except IOError, e:
        raise Failure("Could not copy isi_tools.pth into python site-libs directory. %s"% e)
    except ImportError, e:
        raise Failure("Could not import distutils, broken python installation: %s"% e)

    # this file indicates the package has been installed.
    try:
        f = file(installed_file, "w")
        f.write(target)
        f.close()
    except OSError, e:
        raise Failure("Error writing installation file: %s" % e.strerror)

    os.unlink(filename)
    return

def log(string, level=output_log):
    """Base logging function."""
    if output_level >= level:
        print string

def error(string):
    """Logging function for error-level messages."""
    log(string, output_error)


def set_internal_status(status):
    """Sets the 'internal' status in the hardware versions file."""
    value = ["no", "yes"][ bool(status) ]

    log("Syncing with dongle.")
    (status, output) = run("/usr/bin/isi_dongle_sync")
    if os.WEXITSTATUS(status) != 0:
        raise Failure("dongle sync exited with %d" % (os.WEXITSTATUS(status)))

    if not os.path.exists(hwversions_path):
        lines = []
    else:
        try:
            f = file(hwversions_path)
            text = f.read()
            f.close()
        except OSError, e:
            raise Failure("Error reading hwversions file: %s" % (e.strerror))

        # backup previous version
        try:
            f = file(hwversions_path + ".bak", "w")
            f.write(text)
            f.close()
        except OSError, e:
            raise Failure("Error making hwversions backup: %s" % (e.strerror))

        # remove any previous entry
        lines = [l for l in text.splitlines() if not l.startswith("internal ")]

    lines.append("internal %s\n" % (value))

    try:
        f = file(hwversions_path + ".new", "w")
        f.write( "\n".join(lines) )
        f.close()
        os.rename(hwversions_path + ".new", hwversions_path)
    except OSError, e:
        raise Failure("Error updating hwversions file: %s" % (e.strerror))

    log("Pushing to dongle.")
    (status, output) = run("/usr/bin/isi_dongle_sync -p")
    if os.WEXITSTATUS(status) != 0:
        raise Failure("dongle push exited with %d" % (os.WEXITSTATUS(status)))

    log("Set internal = %s" % value)
    return

def wait_for_network(timeout=300, poll_delay=20):
    """This returns True once the external network is available, or
False if the timeout is reached.
 timeout (0 to disable)     - give up after waiting this long.
 poll_delay                 - how long to sleep between checks."""

    start_time = time.time()
    rv         = True

    while not network_is_up():
        now = time.time()
        if now - start_time > timeout and timeout != 0:
            log("Timed out waiting for external network.")
            rv = False
            break

        log("Waiting for external network.")
        time.sleep(poll_delay)

    return rv


def network_is_up():
    """This returns true if the external network is available."""
    rv = False

    try:
        os.stat("/etc/resolv.conf")
    except OSError:
        return rv

    if pre_flexnet_merge:
        ext_iface = ni.LNIConfig().list(ni.IFFLAG_EXTERNAL)
        for i in ext_iface:
            if i.up:
                rv = True
                break
    else:
        try:
            ext_iface = flx.lni_query(ifs.DEVID_LOCAL, 0, flx.IFFLAG_EXTERNAL)
            for i in ext_iface:
                if i.nic.up:
                    rv = True
                    break
        except flx.Error, e:
            pass

    return rv


def main(argv=None):
    if argv is None:
        argv = sys.argv

    global output_level         # modified by --quiet

    # these variables indicate what task isi_tools was asked to perform
    get_internal    = False     # returns the 'internal' override setting
    install         = False     # controls installation mode
    internal_value  = None      # controls whether to set internal status

    # these modify the behavior of installation
    build           = None      # install the package for a specific build
    fresh_only      = False     # true = only install if there is no
                                #   existing installation
    internal_only   = False     # only install if 'internal' is true
    latest          = False     # install latest tools package for this branch
    target          = installation_dir
    wait            = False     # wait for network

    try:
        try:
            opts, args = getopt.getopt(argv[1:], "b:fghilnqs:t:w",
                ["build=", "fresh-only", "get-internal", "help", "install",
                "internal-only", "latest", "set-internal=", "target=", "wait"])
        except getopt.error, msg:
            raise Usage(msg)

        if not opts:
            raise Usage(__doc__)

        for o, a in opts:
            if o in ("-b", "--build"):
                build = a
            elif o in ("-f", "--fresh-only"):
                fresh_only = True
            elif o in ("-g", "--get-internal"):
                get_internal = True
            elif o in ("-h", "--help"):
                raise Usage(__doc__)
            elif o in ("-i", "--install"):
                install = True
            elif o in ("-l", "--latest"):
                latest = True
            elif o in ("-n", "--internal-only"):
                internal_only = True
            elif o in ("-q", "--quiet"):
                # 0 is the minimum output level and means no output
                if output_level > 0:
                    output_level -= 1
            elif o in ("-s", "--set-internal"):
                if a != "0" and a != "1":
                    raise Usage("invalid arg for --set-internal, valid: 0/1")
                internal_value = int(a)
            elif o in ("-t", "--target"):
                target = a
            elif o in ("-w", "--wait"):
                wait = True
            else:
                raise Usage("Unexpected option %s" % o)


        if not install and not get_internal and internal_value is None:
            raise Usage(__doc__)
        if build is not None and latest:
            raise Usage("--latest is incompatible with --build")

        if get_internal:
            log("This node's internal status is %s" % (hal.isIsilonNode()))
        elif internal_value is not None:
            set_internal_status(internal_value)
        else: # install
            with applib.safelockfile(engtools_lock,'w'):
                install_ok = True
                if fresh_only and os.path.exists(installed_file):
                    log("Not installing; tools package already installed.")
                    install_ok = False
                elif internal_only and not hal.isIsilonNode():
                    log("Not installing; not an Isilon internal node.")
                    install_ok = False

                if install_ok:
                    build_info = get_build_info(build)

                    if wait:
                        ret = wait_for_network()
                        if ret == False:
                            raise Failure("Network was unavailable.")

                    filename = get_tools_tar(latest=latest, build_info=build_info)
                    if filename is None:
                        raise Usage("Could not retrieve tools package")

                    install_tools_tar(filename, target)


    except Usage, e:
        print e.msg
        return 2
    except Failure, e:
        error("Failed: %s" % e.msg)
        return 1

    return 0

if __name__ == "__main__":
    sys.exit(main())

