#!/usr/local/bin/python

# Copyright C 2002-2004 by Isilon Systems, Inc. All rights not expressly
# granted herein are reserved.
# Created in the United States of America.
# 
# This digital media is protected by U.S. and international copyright and
# intellectual property laws. Unless otherwise specified, all information
# and screens appearing as part of this digital medium, including
# software, services, documents, text, images, icons, and logos design;
# the selection, assembly, arrangement, and design thereof; and the code
# that enables its presentation, are the sole property of Isilon Systems,
# Inc.
# 
# Except as otherwise required by applicable law, no part of any
# copyrighted material may be reproduced, modified, translated,
# distributed, published, or transmitted by any means, electronic or
# mechanical, for any purpose, without the prior written consent of Isilon
# Systems. 
#  
# Isilon and Isilon Systems are registered trademarks of Isilon Systems,
# Inc.  Legion is a trademark of Isilon Systems, Inc.  All other
# trademarks and registered trademarks are the property of their
# respective holders.

import getopt
import gzip
import hashlib
import os
import sys
import syslog
import time

import isi.app.lib.procs as procs
import isi.config.arrcfg as arrcfg
import isi.fs.journal as jlib
import isi.hw.hal as hal
import isi.misc

syslog.openlog(os.path.basename(sys.argv[0]), syslog.LOG_PID, syslog.LOG_USER)

def usage():
    print >> sys.stderr, "   usage: isi_save_journal [-f] [-o file]"
    print >> sys.stderr, ""
    print >> sys.stderr, "      -f: force to save journal."
    print >> sys.stderr, " -o file: write the journal to the file."
    print >> sys.stderr, "      -h: help"
    sys.exit(2)

def out(line, log=False):
    print line
    if log:
        syslog.syslog(syslog.LOG_NOTICE, line)
    sys.stdout.flush()
 
def log(line):
    return out(line, True)

def writeSerial(serial, filename, indent=''):
    out('Writing %s %s: %s' % (os.path.basename(filename), indent, serial))

    try:
        f = open(filename, 'w')
        f.write('%s\n' % serial)
        f.flush()
        os.fsync(f.fileno())
        f.close()
    except EnvironmentError, e:
        out('Unable to write %s: %s' % (filename, e))
        return -1

def validate_backup_cmd():
    journal = None
    unmount = False
    lock = None
    error = -1
    try:
        lock = jlib.lock()
        journal = jlib.JournalImdd()
        unmount = not journal.mounted

        (backup_serial, valid_serial) = jlib.backup_serials()
        out('Journal backup serial: %s' % backup_serial)
        out(' Journal valid serial: %s' % valid_serial)

        try:
            jlib.validate_backup(journal, True)
        except OSError, e:
            out("Error while validating journal: %s" % e)
        except jlib.ValidationError, e:
            out("Problem validating journal: %s" % e)
        else:
            out("Journal backup is valid.")
            error = 0
    finally:
        if journal and unmount and journal.mounted and not journal.umount():
            out('ERROR: unable to unmount %s' % journal.mountpoint)

        jlib.unlock(lock)

    return error

def save_journal(force, ofile, serial, disarm):
    """saves and validates the journal
     Errors returned are as follows:
      0 - journal was saved and validated successfully
      1 - there was a failure that prevented the journal from
         being saved or validated.  These failures should be
         interpreted as warnings
     -1 - an error has occurred"""
    journal = None
    lock = None
    ifsUnmounted = False
    unmount = False
    saving_bad = False
    skip_valid = force

    if ofile is None:
        temp = 'default location'
    else:
        temp = ofile
    log('Attempting to save journal to ' + temp)

    if not arrcfg.is_local_configured():
        log("Local node is unconfigured, not saving journal.")
        return 1

    try:
        ac = arrcfg.config_load()
        node = ac.find_device_by_devid(ac.local_devid)
    except Exception, e:
        log("Failed to read/process cluster config, not saving journal.")
        return -1

    if node is None:
        log("Local node (%d) not found in cluster config, "
                "not saving journal" % ac.local_devid)
        return -1

    try:
        nvram_size = hal.getNvramSize()

        # Grab our journal backup lock to make sure we're the
        # only ones mucking with /var/journal and friends
        lock = jlib.lock()

        if isi.misc.ifs_mounted():
            if node.read_only_as_bool or \
                   isi.misc.sysctlbyname('efs.gmp.shutdown_read_only'):
                out("OneFS is mounted read-only")
                jserial = jlib.read_only_serial()
                if serial is None:
                    serial = jserial
                    out(("No serial specified, using current read-only " +
                         "serial: %d") % serial)
                if serial == jserial:
                    # no need to force if the serials match
                    force = False
                else:
                    out(('Specified read-only serial %d does not match ' +
                         'system read-only serial %d') % (serial, jserial))
                    if force:
                        out('Forcing save due to -f flag')
                        out("NOTE: Forced backups are not considered valid "
                            "for automatic journal restores")
                        serial = ('forced serial=%s backup at %s' %
                                  (serial, time.asctime()))
                    else:
                        return 1
            elif force:
                out("OneFS is mounted read/write, forcing save due to " +
                        "-f flag")
                out("NOTE: Forced backups are not considered valid for " +
                    "automatic journal restores")
                # Use a custom force serial make sure it will never
                # match a serial written out by the kernel
                serial = 'forced backup at %s' % time.asctime()
            else:
                out('OneFS is mounted read/write. Please unmount or switch ' +
                    'to read-only mode')
                out('before attempting a journal backup.')
                return 1
        else:
            # If the journal is bad we'll still attempt a save if
            # we don't have a valid backup.  The reasoning here is
            # that any backup is better than none.
            if os.path.exists(jlib.FILES['bad']):
                log('Warning: %s exists. Saving bad journal.' % \
                    jlib.FILES['bad'])
                saving_bad = True

            # OneFS is not mounted. We'll need to write out both
            # the normal serial file and the serial validation
            # file ourselves (using a special unmounted serial
            # that won't conflict with the standard read-only
            # serials), since OneFS isn't around to do the latter
            # for us
            out('OneFS is unmounted')
            ifsUnmounted = True
            serial = 'unmounted backup at %s' % time.asctime()

        if ofile is None:
            journal = jlib.JournalImdd()
            ofile = os.path.join(journal.mountpoint, "journal")
 
            # Flash upgrade from a Tabasco or earlier releases.
            if not os.path.exists(journal.mountpoint):
                return 1

            unmount = not journal.mounted

            try:
                jlib.validate_backup(journal, True)
                # We're checking to make sure we're not saving over
                # a valid backup, so we expect validation to fail.
                # Don't pass these exceptions on.
            except OSError, e:
                pass
            except jlib.ValidationError, e:
                pass
            else:
                if skip_valid:
                    log('Warning, saving over valid journal backup')
                else:
                    log('A valid backup journal already exists. Not saving.')
                    return 1

            if journal.mounted:
                if not journal.umount():
                    out('ERROR: unable to unmount %s' % journal.mountpoint)
            else:
                unmount = True

            # Run newfs.
            cmd = '/sbin/newfs -i 4096 %s' % journal.mountdev
            newfs_err, newfs_out = procs.get_cmd_output(cmd)
            if newfs_err:
                log('ERROR: newfs returned error code %s, output is %s' %
                    (newfs_err, newfs_out))

            if not journal.mount():
                log('ERROR: unable to mount %s at %s' %
                    (journal.mountdev, journal.mountpoint))
                return -1

            user_specified = False
            valid_file = jlib.FILES['backup_valid']
            serial_file = jlib.FILES['backup_serial']
        else:
            # If the user specified a path, then we can't modify the default
            # journal backup valid file (in /etc/ifs/journal_backup_valid).
            # Instead, we write an equivalent _valid file at their specified
            # path.
            user_specified = True
            valid_file = ofile + ".valid"
            serial_file = ofile + ".serial"

        # Write out the new serial file. This should invalidate
        # any existing backup, since serials are more-or-less
        # unique, but we first write a 0 to the valid file
        # just to make sure.
        if writeSerial('backup in progress', valid_file, ' '):
            log('Failed to write valid file "' + valid_file +
                '", unable to save journal.')
            return -1
        if writeSerial(serial, serial_file):
            log('Failed to write out serial file "' + serial_file +
                '", unable to save journal')
            return -1

        try:
            read_size = gzip_journal(ofile)
        except Exception, e:
            log("Unable to save journal %s: %s" % (ofile, e))
            return -1

        if disarm:
            try:
                jlib.nvr_disarm_flash()
            except Exception, e:
                # Don't error out if the disarm fails. As long as the above
                # save to disk succeeded, we're safe leaving the
                # save-to-flash armed since we always prefer the disk
                # backup over the flash backup. See isi_testjournal's
                # recover_journal() for details. Squawk but continue.
                log("Unable to disarm save-to-flash: %s" % str(e))

        if ifsUnmounted:
            # Unmounted journal saves are always valid (assuming ifs is
            # still unmounted), but have to be validated by us, since
            # /ifs isn't around to do it for us
            if isi.misc.ifs_mounted():
                log('OneFS is now mounted, marking backup journal invalid')
                serial = 'invalid %s' % serial
            if writeSerial(serial, valid_file, indent=' '):
                return 1
        elif force:
            # Forced journal saves are never valid, so mark this one as such
            if writeSerial('invalid %s' % serial, valid_file, indent=' '):
                return 1
        elif saving_bad:
            # Forced save while journal is bad, so mark this one as bad
            if writeSerial('bad %s' % serial, valid_file, indent=' '):
                return -1
        elif not user_specified:
            # Journal saves while in read-only mode have to be validated
            # by the kernel
            out('Requesting journal backup validation from kernel')
            try:
                jlib.ifs_validate_journal_backup(serial)
            except OSError, e:
                if e.errno == 80: # EAUTH, which isn't found in errno module
                    log('WARNING: Kernel refused validation, backup ' +
                        'journal marked invalid')
                    return 1
                else:
                    log('ERROR: Kernel validation failed, ' +
                        os.strerror(e.errno))
                    raise e

        log("All data saved successfully")
        return 0
    finally:
        if journal and unmount and journal.mounted and not journal.umount():
            log('ERROR: unable to unmount %s' % journal.mountpoint)

        jlib.unlock(lock)

class Saver(object):
    """Class that manages saving journal data generated by
       isi_dumpjournal -s. Data is simultaneously gzipped and
       md5'd as it's generated.
       """
       
    def __init__(self, ofile):
        self._gzName = ofile + ".gz"
        self._md5Name = ofile + ".md5"
        self._gzTemp = self._gzName + ".temp"

        self._size = 0
        self._md = hashlib.md5()
        self._g = gzip.GzipFile(filename=self._gzTemp, mode="wb")
        self._errtext = ""

    def _proc_stdout(self, data):
        # Journal data comes from stdout. Add it to the md5 tally,
        # as well as the gzip stream.
        self._size += len(data)
        self._md.update(data)
        self._g.write(data)

    def _proc_stderr(self, data):
        # Progress text goes to stderr. This can be quite large
        # if there's a lot of journal data, so we don't print
        # it out by default. It can be handy to see sometimes,
        # though.

        self._errtext += data;

        # Comment out this line to print output text to stdout
        return

        print data.rstrip()
        sys.stdout.flush()

    def save(self):
        # Save the journal
        log('Saving journal to %s' % self._gzName)
        cmd = '/usr/bin/isi_dumpjournal -s - -f %s' % hal.getJournalDevicePath()
        error = procs.proc_cmd_no_output(cmd, self._proc_stdout,
                          include_stderr=False, err_func=self._proc_stderr, 
                          line_based=False)
        if error:
            lines = self._errtext.splitlines()
            print "Unable to save journal (isi_dumpjournal -s returned %d)" \
                % error
            print "Error output:"
            print "\t"+"\n\t".join(lines[-5:])

            self._g.close();
            os.unlink(self._gzTemp)
            raise IOError

        # Write the md5 file
        md5File = open(self._md5Name, 'w')
        md5File.write(self._md.hexdigest())
        md5File.write('\n')
        md5File.flush()
        os.fsync(md5File.fileno())
        md5File.close()

        # Make sure the journal contents actually get to the disk
        self._g.flush()
        os.fsync(self._g.fileobj.fileno())
        self._g.close()

        # Move the journal to the correct place
        os.rename(self._gzTemp, self._gzName)

        return self._size

def gzip_journal(ofile, device='/dev/mnv0'):
    return Saver(ofile).save()
    
if __name__ == "__main__":
    try:
        opts, args = getopt.getopt(sys.argv[1:], "fho:s:",
                ["validate", "disarm"])
    except:
        usage()

    if (not len(args) == 0):
        usage()

    force = False
    ofile = None
    serial = None
    v = False
    disarm = False
    for opt in opts:
        if opt[0] == '-h':
            usage()
        elif opt[0] == '-f':
            force = True
        elif opt[0] == '-o':
            ofile = opt[1]
        elif opt[0] == '--validate':
            v = True
        elif opt[0] == '--disarm':
            disarm = True
        elif opt[0] == '-s':
            try:
                serial = long(opt[1])
            except ValueError, e:
                out('ERROR: %s' % e)
                usage()
    error = 0
    if v:
        try:
            error = validate_backup_cmd()
        except:
            error = -1
    else:
        error = save_journal(force, ofile, serial, disarm)

    sys.exit(error)
