#!/usr/bin/env python
#
# This utility is used to enable and configure cluster-wide small file packing
# in OneFS. Filepool policies can be configured through isi filepool as usual.
# The settings configured here are to enable/disable features and set limits.
#
# Note that this utility is intentionally not helpful, for reasons.
#

import argparse
import os
import subprocess
import sys

import isi.app.lib.confParser as confParser
import isi.fs.pctl2 as pctl2
import isi.gconfig as gconfig
import isi.misc as misc
import isi.papi.basepapi as basepapi

sys.path.insert(0, '/usr/local/share/CLI')
import prompt
import views.lang as lang
import views.valueformatter as vf


ENABLED_SYSCTL                    = 'efs.sfm.pack.enabled'
ADS_ENABLED_SYSCTL                = 'efs.sfm.pack.ads_enabled'
SNAPS_ENABLED_SYSCTL              = 'efs.sfm.pack.snaps_enabled'
MIRROR_CONTAINERS_ENABLED_SYSCTL  = 'efs.sfm.pack.mirror_containers_enabled'
MIRROR_TRANSLATION_ENABLED_SYSCTL = 'efs.sfm.pack.mirror_translation_enabled'
UNPACK_RECENTLY_MODIFIED_SYSCTL   = 'efs.sfm.pack.unpack_recently_modified'
UNPACK_SNAPS_SYSCTL               = 'efs.sfm.pack.unpack_snaps'
AVOID_DEDUPED_FILES_SYSCTL        = 'efs.sfm.pack.avoid_bsin'
MAX_SIZE_SYSCTL                   = 'efs.sfm.pack.max_size'
SIN_CACHE_CUTOFF_SIZE_SYSCTL      = 'efs.sfm.pack.sin_cache_cutoff_size'
MIN_AGE_MTIME_SYSCTL              = 'efs.sfm.pack.min_age_mtime'
DIR_HINT_ENTRIES_SYSCTL           = 'efs.sfm.pack.dir_hint_entries'
CONTAINER_MIN_SIZE_SYSCTL         = 'efs.sfm.pack.container_min_size'
CONTAINER_MAX_SIZE_SYSCTL         = 'efs.sfm.pack.container_max_size'


SYSCTL_CONF_PATH    = '/etc/mcp/override/sysctl.conf'
LICENSE_PATH        = '/LICENSE.txt'
SYSCTL_COMMENT      = '# set by isi_packing'

# Intentionally do not provide any help
class UnhelpfulParser(argparse.ArgumentParser):
    def print_usage(self, file):
        pass


def display_settings():
    print 'Enabled:\t\t\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(ENABLED_SYSCTL))
    print 'Enable ADS:\t\t\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(ADS_ENABLED_SYSCTL))
    print 'Enable snapshots:\t\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(SNAPS_ENABLED_SYSCTL))
    print 'Enable mirror containers:\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(
        MIRROR_CONTAINERS_ENABLED_SYSCTL))
    print 'Enable mirror translation:\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(
        MIRROR_TRANSLATION_ENABLED_SYSCTL))
    print 'Unpack recently modified:\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(
        UNPACK_RECENTLY_MODIFIED_SYSCTL))
    print 'Unpack snapshots:\t\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(
        UNPACK_RECENTLY_MODIFIED_SYSCTL))
    print 'Avoid deduped files:\t\t' + \
        vf.TypeFormatter(bool).format(misc.sysctlbyname(
        AVOID_DEDUPED_FILES_SYSCTL))
    print 'Maximum file size:\t\t' + \
        vf.SizeFormatter().format(misc.sysctlbyname(MAX_SIZE_SYSCTL))
    print 'SIN cache cutoff size:\t\t' + \
        vf.SizeFormatter().format(misc.sysctlbyname(
        SIN_CACHE_CUTOFF_SIZE_SYSCTL))
    print 'Minimum age before packing:\t' + \
        vf.DurationFormatter(display_zero=True).format(
        misc.sysctlbyname(MIN_AGE_MTIME_SYSCTL))
    print 'Directory hint maximum entries:\t%u' % \
        misc.sysctlbyname(DIR_HINT_ENTRIES_SYSCTL)
    print 'Container minimum size:\t\t' + \
        vf.SizeFormatter().format(misc.sysctlbyname(
        CONTAINER_MIN_SIZE_SYSCTL))
    print 'Container maximum size:\t\t' + \
        vf.SizeFormatter().format(misc.sysctlbyname(
        CONTAINER_MAX_SIZE_SYSCTL))


def display_fsa(jobid=None):
    try:
        if jobid is None:
            response = basepapi.get('/3/fsa/results')
            for job in response.body['results']:
                if job['job_state'][1] != 'STATE_FINISHED':
                    continue
                if job['id'] > jobid:
                    jobid = job['id']
    except:
        sys.exit('Error: Failed to get most recent jobid')

    if jobid is None:
        sys.exit('Error: No FSAnalyze job results are available')

    try:
        response = basepapi.get('/3/fsa/results/%s/directories' % jobid)
        logical_size = response.body['total_usage']['log_size_sum']
    except:
        sys.exit('Error: Failed to get result from FSAnalyze jobid %s' % jobid)

    try:
        ctx = gconfig.open(gconfig.gcfg_fsaruns).ctx_new()
        results = ctx.read('')['jobs']
        result = filter(lambda x: x['id'] == jobid, results)[0]
        free_blocks = result['fs_blocks_free']
        total_blocks = result['fs_blocks_total']
    except:
        sys.exit('No usable FSAnalyze job result was found.')

    # Found a bad result, probably before upgrade was committed
    if total_blocks == 0:
        sys.exit('No usable FSAnalyze job result was found.')

    if (free_blocks < total_blocks):
        used_size = (total_blocks - free_blocks) * 8192;
        efficiency = 100.0 * logical_size / used_size;
    elif (free_blocks > total_blocks):
        sys.exit('Internal error: free_blocks > total_blocks')
    else:
        used_size = 0
        efficiency = None

    print "FSAnalyze job: %s (%s)" % (jobid,
        vf.UnixTimestampFormatter().format(result['end_time']))
    print "Logical size:  %s" % vf.SizeFormatter().format(logical_size)
    print "Physical size: %s" % vf.SizeFormatter().format(used_size)
    print "Efficiency:    %s" % vf.PercentFormatter().format(efficiency)


language = lang.Lang()

parser = UnhelpfulParser(description=None, add_help=False)
parser.add_argument('--ls', dest='ls',
    help='Show current packing settings',
    action='store_true', default=False)
parser.add_argument('--fsa', dest='fsa',
    help='Show most recent efficiency analysis',
    action='store_true', default=False)
parser.add_argument('--fsa-jobid', dest='fsa_jobid',
    type=long, help=argparse.SUPPRESS)
parser.add_argument('--enabled', dest='enabled',
    metavar='true|false',
    help='Enable packing',
    type=vf.TypeFormatter(bool).parse)
parser.add_argument('--enable-ads', dest='ads',
    metavar='true|false',
    help='Enable packing ADS streams',
    type=vf.TypeFormatter(bool).parse)
parser.add_argument('--enable-snaps', dest='snaps',
    metavar='true|false',
    help='Enable packing snapshot version files',
    type=vf.TypeFormatter(bool).parse)
parser.add_argument('--enable-mirror-containers', dest='mirror_containers',
    metavar='true|false',
    help='Allow containers with mirror protection policies',
    type=vf.TypeFormatter(bool).parse)
parser.add_argument('--enable-mirror-translation', dest='mirror_translation',
    metavar='true|false',
    help='Pack mirrored files into FEC containers with equivalent protection',
    type=vf.TypeFormatter(bool).parse)
parser.add_argument('--unpack-recent', dest='unpack_recent',
    metavar='true|false',
    help='Unpack recently modified files',
    type=vf.TypeFormatter(bool).parse);
parser.add_argument('--unpack-snaps', dest='unpack_snaps',
    metavar='true|false',
    help='Unpack packed snapshot version files',
    type=vf.TypeFormatter(bool).parse);
parser.add_argument('--avoid-dedupe', dest='avoid_dedupe',
    metavar='true|false',
    help='Avoid packing files which are partially deduped',
    type=vf.TypeFormatter(bool).parse);
parser.add_argument('--max-size', dest='max_size', metavar='bytes',
    help='Maximum size of files which may be packed',
    type=vf.SizeFormatter().parse)
parser.add_argument('--sin-cache-cutoff-size', dest='sin_cache_cutoff_size',
    metavar='bytes',
    help='Maximum size of a container shadowstore in cache',
    type=vf.SizeFormatter().parse)
parser.add_argument('--min-age', dest='min_age', metavar='seconds',
    help='Minimum time since modification before packing',
    type=vf.DurationFormatter().parse)
parser.add_argument('--dir-hint-entries', dest='dir_hint_entries',
    metavar='entries',
    help='Number of entries in the directory container hint',
    type=long)
parser.add_argument('--container-min-size', dest='container_min_size',
    metavar='bytes',
    help='Minimum size of container shadow file',
    type=vf.SizeFormatter().parse);
parser.add_argument('--container-max-size', dest='container_max_size',
    metavar='bytes',
    help='Maximum size of container shadow file',
    type=vf.SizeFormatter().parse);

parser.add_argument('--isilon-only', '-I', dest='internal',
    action='store_true', default=False, help=argparse.SUPPRESS)
parser.add_argument('--help', '-h', dest='help',
    action='store_true', default=False, help=argparse.SUPPRESS)
args = parser.parse_args()

if args.help and not args.internal:
    sys.exit('Use of this command is for authorized use only.')

if args.help and args.internal:
    sys.exit(parser.format_help())

modifier_args = [args.enabled, args.ads, args.snaps, args.mirror_containers,
    args.mirror_translation, args.unpack_recent, args.unpack_snaps,
    args.avoid_dedupe, args.max_size, args.sin_cache_cutoff_size, args.min_age,
    args.dir_hint_entries, args.container_min_size, args.container_max_size]

if all(arg is None for arg in modifier_args) and not any([args.ls, args.fsa]):
    sys.exit(0)

if args.ls and any(modifier_args):
    sys.exit('Error: --ls may not be used with any other option')

if args.fsa and any(modifier_args):
    sys.exit('Error: --fsa may not be used with any other option')

# These must be enforced here -- the kernel enforces it but MCP hides the error

if args.max_size is not None and (args.max_size < 0 or
        args.max_size > pctl2.PACK_MAX_MAX_SIZE):
    sys.exit('Error: --max-size must be 0 to %s bytes' %
        vf.SizeFormatter().format(pctl2.PACK_MAX_MAX_SIZE))

if args.sin_cache_cutoff_size is not None and (args.sin_cache_cutoff_size < 0 or
        args.sin_cache_cutoff_size > pctl2.DEFAULT_MAX_SHADOW_SIZE):
    sys.exit('Error: --sin-cache-cutoff-size must be 0 to %s bytes' %
        vf.SizeFormatter().format(pctl2.DEFAULT_MAX_SHADOW_SIZE))

if args.min_age is not None and args.min_age < 0:
    sys.exit('Error: --min-age must be 0 or greater seconds')

if args.dir_hint_entries is not None and (args.dir_hint_entries < 1 or
        args.dir_hint_entries > pctl2.DIR_PACK_HINT_ENTRIES_MAX):
    sys.exit('Error: --dir-hint-entries must be 0 to %u' %
        pctl2.DIR_PACK_HINT_ENTRIES_MAX)

if args.container_min_size is not None and (args.container_min_size < 0 or
        args.container_min_size > pctl2.PACK_MAX_MAX_SIZE):
    sys.exit('Error: --container-min-size must be 0 to %s' %
        vf.SizeFormatter().format(pctl2.PACK_MAX_MAX_SIZE))

if args.container_max_size is not None and \
        (args.container_max_size < pctl2.PACK_MAX_MAX_SIZE or
        args.container_max_size > pctl2.DEFAULT_MAX_SHADOW_SIZE):
    sys.exit('Error: --container-max-size must be %s to %s' % \
        (vf.SizeFormatter().format(pctl2.PACK_MAX_MAX_SIZE),
        vf.SizeFormatter().format(pctl2.DEFAULT_MAX_SHADOW_SIZE)))

if args.ls:
    display_settings()
    exit(0)

if args.fsa:
    display_fsa(args.fsa_jobid)
    exit(0)

if args.internal is not True:
    print 'WARNING: Modifying packing variables may adversely affect the'
    print 'performance of OneFS.'
    if not prompt.confirm(language.ARE_YOU_SURE):
        sys.exit('No changes made.')

if args.enabled is True and args.internal is not True:
    os.environ['LESSSECURE'] = '1'
    subprocess.check_call(['/usr/bin/less', LICENSE_PATH])
    os.unsetenv('LESSSECURE')
    if not prompt.confirm(language.LICENSE_AGREEMENT_PROMPT):
        sys.exit('No changes made.')

conf = confParser.ConfParser(SYSCTL_CONF_PATH)
if args.enabled is not None:
    conf.add_or_modify_value(ENABLED_SYSCTL,
    str(int(args.enabled)), SYSCTL_COMMENT)
if args.ads is not None:
    conf.add_or_modify_value(ADS_ENABLED_SYSCTL,
    str(int(args.ads)), SYSCTL_COMMENT)
if args.snaps is not None:
    conf.add_or_modify_value(SNAPS_ENABLED_SYSCTL,
    str(int(args.snaps)), SYSCTL_COMMENT)
if args.mirror_containers is not None:
    conf.add_or_modify_value(MIRROR_CONTAINERS_ENABLED_SYSCTL,
    str(int(args.mirror_containers)), SYSCTL_COMMENT)
if args.mirror_translation is not None:
    conf.add_or_modify_value(MIRROR_TRANSLATION_ENABLED_SYSCTL,
    str(int(args.mirror_translation)), SYSCTL_COMMENT)
if args.unpack_recent is not None:
    conf.add_or_modify_value(UNPACK_RECENTLY_MODIFIED_SYSCTL,
    str(int(args.unpack_recent)), SYSCTL_COMMENT)
if args.unpack_snaps is not None:
    conf.add_or_modify_value(UNPACK_SNAPS_SYSCTL,
    str(int(args.unpack_snaps)), SYSCTL_COMMENT)
if args.avoid_dedupe is not None:
    conf.add_or_modify_value(AVOID_DEDUPED_FILES_SYSCTL,
    str(int(args.avoid_dedupe)), SYSCTL_COMMENT)
if args.max_size is not None:
    conf.add_or_modify_value(MAX_SIZE_SYSCTL,
    str(args.max_size), SYSCTL_COMMENT)
if args.sin_cache_cutoff_size is not None:
    conf.add_or_modify_value(SIN_CACHE_CUTOFF_SIZE_SYSCTL,
    str(args.sin_cache_cutoff_size), SYSCTL_COMMENT)
if args.min_age is not None:
    conf.add_or_modify_value(MIN_AGE_MTIME_SYSCTL,
    str(args.min_age), SYSCTL_COMMENT)
if args.dir_hint_entries is not None:
    conf.add_or_modify_value(DIR_HINT_ENTRIES_SYSCTL,
    str(args.dir_hint_entries), SYSCTL_COMMENT)
if args.container_min_size is not None:
    conf.add_or_modify_value(CONTAINER_MIN_SIZE_SYSCTL,
    str(args.container_min_size), SYSCTL_COMMENT)
if args.container_max_size is not None:
    conf.add_or_modify_value(CONTAINER_MAX_SIZE_SYSCTL,
    str(args.container_max_size), SYSCTL_COMMENT)
conf.write_and_rename(SYSCTL_CONF_PATH + '.old')

