#!/usr/bin/env python
# coding: utf-8
#
# Copyright (c) 2021-2024 Dell Inc. or its subsidiaries. All Rights Reserved.
#
"""
Description:
    Command Line Utility tool to run, view HealthCheck evaluation results
    leveraging Isilon HealthCheck Framework
"""
from __future__ import print_function

import argparse
import concurrent.futures
import datetime
import getpass
import json
import logging.handlers
import os
import platform
import re
import socket
import sys
import textwrap
import time
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import wait

IS_POWERSCALE = "Isilon" in platform.system()
from operator import itemgetter

import ssl
import pickle

if IS_POWERSCALE:
    import isi.papi.basepapi as basepapi
    from isi.app.lib.emailer import Emailer
else:
    import requests

    LOGSET_WORKERS = 6
    import certifi
    import smtplib
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText

TOOL_VERSION = "1.0"

PAPI_HC_ROOT = "7/healthcheck/"
PAPI_HC_CHKLIST = PAPI_HC_ROOT + "checklists/"
PAPI_HC_ITEMS = PAPI_HC_ROOT + "items/"
PAPI_HC_EVALUATIONS = PAPI_HC_ROOT + "evaluations/"

PAPI_UPGRADE_PATCHES = "7/upgrade/cluster/patch/patches"
PAPI_CLUSTER_CONFIG = "3/cluster/config"
PAPI_CLUSTER_EMAIL = "1/cluster/email"

COLOR_OK = "\033[92m"  # Green
COLOR_WARN = "\033[93m"  # Orange
COLOR_ERR = "\033[91m"  # Red
COLOR_OKBLUE = "\033[94m"  # Ok Blue
COLOR_GRAY = "\033[97m"  # Ok Gray
FORMAT_END = "\033[0m"  # end of special formatting
STYLE_BOLD = "\033[1m"
STYLE_UL = "\033[4m"

MSG_PASSED = "Passed"
MSG_FAILED = "Failed"

STATUS_UNS = "UNSUPPORTED"
EMAIL_FROM_ADDRESS = "isihealthcheck@%s" % socket.getfqdn()
EMAIL_SUBJECT = "PowerScale Health Check Results"

CHECKS_BY_DOMAIN = {
    "admin": ("Admin", ["avscan", "celog", "ntp", "cluster_capacity", "log_level"]),
    "backup": ("Backup", ["synciq", "ndmp", "snapshot"]),
    "filesystem": ("FileSystem", ["cluster_capacity", "job_engine", "smartpools"]),
    "health": ("Health", ["basic", "security", "quick_check"]),
    "network": ("Network", ["infiniband", "smartconnect"]),
    "performance": ("Performance", ["performance"]),
    "protonix": ("ProtoNIX", ["auth", "nfs"]),
    "protowin": ("ProtoWIN", ["auth", "smb"]),
    "upgrade": ("Upgrade", ["pre_upgrade", "post_upgrade"]),
}


logger = logging.getLogger("isi_healthcheck_tool")
logger.setLevel(logging.INFO)


def display_success(message):
    return COLOR_OK + message + FORMAT_END


def display_failure(message):
    return COLOR_ERR + message + FORMAT_END


def display_warn(message):
    return COLOR_WARN + message + FORMAT_END


def display_bold(message):
    return STYLE_BOLD + message + FORMAT_END


def display_ul(message):
    return STYLE_UL + message + FORMAT_END


if not IS_POWERSCALE:
    if "OFHC_API" not in os.environ:
        logger.error(
            display_failure(
                "Please set OFHC_API environment variable. Example export OFHC_API=<IP>:8001/api/1.0.0/"
            )
        )
        os._exit(1)
    OFHC_API = os.environ["OFHC_API"]
    """if "OFHC_CERT" not in os.environ:
        logger.error(display_failure("Please set OFHC_CERT environment variable. Example export OFHC_CERT=/root/python-docker/ca.pem"))
        os._exit(1)
    OFHC_CERT = os.environ["OFHC_CERT"]"""


def get_color_coded_status(item):
    cli = ""
    html = ""
    if item["passed"]:
        if item["status"] == "OK":
            cli = COLOR_OK + "PASSED" + FORMAT_END
            html = "#00FF00"
        elif item["status"] == STATUS_UNS:
            cli = COLOR_GRAY + "PASSED" + FORMAT_END + " (" + item["status"] + ")"
            html = "#C0C0C0"
        else:
            cli = COLOR_OKBLUE + "PASSED" + FORMAT_END + " (" + item["status"] + ")"
            html = "#0000FF"
    else:
        if item["status"] == "WARNING":
            cli = COLOR_WARN + "FAILED (" + item["status"] + ")" + FORMAT_END
            html = "#FFA500"
        else:
            cli = COLOR_ERR + "FAILED (" + item["status"] + ")" + FORMAT_END
            html = "#FF0000"
    return (cli, html)


"""
Utility class to communicate with HealthCheck Framework through PAPI intergace.
This class leverages HealthCheck Framework to execute commands and present the
results in user-friently manner
"""


class HealthCheckEvaluation:
    def __init__(
        self,
        checknames,
        show_passed,
        group_by_nodes,
        show_unsupported,
        logset,
        filepath=None,
        email_addresses=None,
        bad_request=False,
    ):
        # do some hygiene work and remove duplicates etc
        self.eval_names = None
        self.checknames = checknames

        self.checknames = checknames
        self.show_passed = show_passed
        self.group_by_nodes = group_by_nodes
        self.show_unsupported = show_unsupported
        self.logset = logset
        self.bad_request = bad_request
        self.filepath = filepath
        self.email_addresses = email_addresses
        self.eval_results = None

        if not checknames or len(checknames) == 0:
            logger.error("Checklist name not specified (empty)")
            os._exit(1)

        if checknames:
            checks = checknames.strip().split(",")
            checks = [x.strip() for x in checks]
            self.eval_names = list(set(checks))

        if not self.eval_names:
            logger.error("Specified checklist names are not valid")
            os._exit(1)

        if email_addresses:
            addresses = email_addresses.strip().split(",")
            addresses = [x.strip() for x in addresses]
            self.eval_addesses = list(set(addresses))

    def submit_evals(self, logset, override, priority, check_parameters):
        """
        Wrapper method to submit Healcheck evaluations
        :param checks: list of checks to be evaluated
        :return a dictionary with {checkname: ID} for status monitoring
        """
        if not IS_POWERSCALE:
            found = {}
            req_checklists = self.eval_names
            installed_checklists_list = get_checklist()
            installed_checklists = []
            for checklist_item in installed_checklists_list:
                installed_checklists.append(checklist_item["id"])

            checklists = []
            for checklist in req_checklists:
                found = True
                if checklist not in installed_checklists:
                    installed_checks = list(get_check_details().keys())
                    if checklist not in installed_checks:
                        found = False
                if found:
                    checklists.append(checklist)
                else:
                    err_msg = (
                        (
                            "Specified check '{0}' either not supported in this"
                            + " release or not found. Skipping this check..."
                        )
                    ).format(checklist)
                    logger.error(display_failure(err_msg))
                    logger.debug(err_msg)

            checklists = ",".join(checklists)
            if checklists == "":
                return {}, False

            max_retry_count = 3
            script_params = {}
            if check_parameters and len(check_parameters) % 2 == 0:
                for index, key in enumerate(check_parameters):
                    if index % 2 == 0:
                        script_params.update({key: check_parameters[index + 1]})

            parameters = {}
            parameters["override_cached_responses"] = override
            parameters["priority"] = priority
            parameters["check_parameters"] = script_params
            bad_request = False
            logger.debug("Starting evaluations")
            retry_count = 0
            while retry_count < max_retry_count:
                retry_count += 1
                request_body = {
                    "name": getpass.getuser(),
                    "checklists": checklists,
                    "logset_path": logset,
                    "parameters": parameters,
                }

                response = requests.post(OFHC_API + "jobs", json=request_body)
                eval_requests = {}
                if response.status_code == 200:
                    eval_id = response.json()["eval_id"]
                    for check in checklists.split(","):
                        eval_requests[check] = eval_id
                    break
                if response.status_code == 400:
                    logger.error(display_failure(response.json()))
                    bad_request = True
                    break
                else:
                    # it does not allow same check run within same minutes
                    time.sleep(60 - time.gmtime().tm_sec + 1)

            request_id = ",".join(k + v for k, v in eval_requests.items())
            if len(eval_requests) > 0:
                logger.info(
                    (
                        "\nInitiated Cluster health check evaluations(ref Id: '{0}')"
                    ).format(request_id)
                )
                print("Monitoring evaluation progresss, please wait.", end="")
                sys.stdout.flush()
            return eval_requests, bad_request

        uri = PAPI_HC_EVALUATIONS
        param = {}
        overrides = []

        eval_requests = {}
        max_retry_count = 3
        logger.debug("Starting evaluations")
        for check in self.eval_names:
            retry_count = 0
            while retry_count < max_retry_count:
                retry_count += 1
                request_body = {
                    "checklist_id": check,
                    "parameters": param,
                    "overrides": overrides,
                }
                response = basepapi.post(uri, body=request_body)
                if response.status == 201:
                    # evaluation request submitted successfully
                    eval_id = response.body["id"]
                    eval_requests[check] = eval_id
                    break
                elif response.status == 400:
                    # it does not allow same check run within same minutes
                    time.sleep(60 - time.gmtime().tm_sec + 1)
                else:
                    err_msg = (
                        (
                            "Specified check '{0}' either not supported in this"
                            + " release or not found. Skipping this check..."
                        )
                    ).format(check)
                    logger.error(display_failure(err_msg))
                    logger.debug("API response: '%s'", response)
                    break

        request_id = ",".join(eval_requests.values())
        if len(eval_requests) > 0:
            logger.info(
                ("Initiated Cluster health check evaluations(ref Id: '{0}')").format(
                    request_id
                )
            )
            print("Monitoring evaluation progresss, please wait.", end="")
            sys.stdout.flush()

        return eval_requests, False

    def is_item_displayable(self, check_result):
        """
        Check whether a check result is dislayable based on choosen options
        """
        if check_result["passed"]:
            if check_result["status"] == STATUS_UNS:
                return self.show_unsupported
            else:
                return self.show_passed
        else:
            return True

    def show_check_result(self, check_result, include_node=False, node_ids=None):
        """
        Show HealthCheck results to console
        """
        if not self.is_item_displayable(check_result):
            return

        if include_node and check_result["node"] and check_result["node"] != "cluster":
            node_txt = node_ids if node_ids else check_result["node"]
            node_txt += ")"
            logger.info(
                "  {0:<35} lnn({1:<10}  {2:<10}".format(
                    check_result["name"],
                    node_txt,
                    get_color_coded_status(check_result)[0],
                )
            )
        else:
            logger.info(
                "  {0:<50}  {1:<10}".format(
                    check_result["name"], get_color_coded_status(check_result)[0]
                )
            )

        if (not check_result["passed"]) or (
            check_result["passed"] and check_result["status"] == "INFO"
        ):
            fail_details = check_result["details"]
            if isinstance(fail_details, (str)):
                fail_details = [fail_details]
            elif isinstance(fail_details, (dict)):
                fail_details = [
                    "%s: %s" % (k, v) for (k, v) in list(fail_details.items())
                ]

            # Its array of details, wrap them and display
            wrapper = textwrap.TextWrapper(width=70)
            for detail in fail_details:
                if isinstance(detail, str):
                    lines = wrapper.wrap(text=detail)
                    for line in lines:
                        logger.info("  {0:<50}  {1:<72}".format("", line))
                else:
                    lines = json.dumps(detail, indent=2).split("\n")
                    for line in lines:
                        wraps = wrapper.wrap(text=line)
                        for ln in wraps:
                            logger.info("  {0:<50}  {1:<72}".format("", ln))
            logger.info("")

    @staticmethod
    def show_eval_summary_stats(eval_result, header=False, historical=False):
        """
        Show HealthCheck summary stats to console
        """
        evaltime = time.strftime("%x %X", time.localtime(eval_result["start_time"]))
        eval_results = eval_result["details"]
        eval_name = eval_result["checklist_id"]
        if IS_POWERSCALE:
            eval_id = eval_result["id"]
        else:
            eval_id = eval_result["eval_id"]
        fail_count = 0
        us_cnt = 0
        for result in eval_results:
            if not result["passed"]:
                fail_count += 1
            elif result["status"] == STATUS_UNS:
                us_cnt += 1

        if header:
            logger.info("Evaluation summary for check '%s'", eval_name)
            logger.info("{0:^20}{1:^20} {2:^25}".format("ID", "Timestamp", "Result"))
            logger.info("{0:=<65}".format(""))

        if fail_count > 0:
            if historical:
                result = "{0}({1}), Unsupported({2}), {3}({4})".format(
                    display_success(MSG_PASSED),
                    len(eval_results) - fail_count - us_cnt,
                    us_cnt,
                    display_failure(MSG_FAILED),
                    fail_count,
                )
                logger.info("{0:<20} {1:<20} {2:<30}".format(eval_id, evaltime, result))
            else:
                logger.info(
                    (
                        "Evaluation summary for check '%s': [%s] %s(%d), "
                        + "Unsupported(%d), %s(%d)"
                    ),
                    eval_name,
                    evaltime,
                    display_success(MSG_PASSED),
                    len(eval_results) - fail_count - us_cnt,
                    us_cnt,
                    display_failure(MSG_FAILED),
                    fail_count,
                )
        else:
            if historical:
                logger.info(
                    "%s: [%s] %s(%d), Unsupported(%d)",
                    eval_id,
                    evaltime,
                    display_success(MSG_PASSED),
                    len(eval_results) - us_cnt,
                    us_cnt,
                )
            else:
                logger.info(
                    (
                        "Evaluation summary for check '%s': [%s] %s(%d), "
                        + "Unsupported(%d)"
                    ),
                    eval_name,
                    evaltime,
                    display_success(MSG_PASSED),
                    len(eval_results) - us_cnt,
                    us_cnt,
                )
        logger.info("")  # add a newline

    @staticmethod
    def format_node_lnns(node_ids):
        """
        Format node lnns in decorated string format
        """
        id_str = str("")
        if not node_ids:
            return id_str
        sorted_ids = sorted(list(set(node_ids)))
        prev = None
        begin = None
        cont_cnt = 0

        for id in sorted_ids:
            if prev is None:
                prev = id
                begin = id
            elif prev != id:
                if id == prev + 1:
                    cont_cnt += 1
                    prev = id
                else:
                    if cont_cnt > 0:
                        id_str += "{0}-{1},".format(begin, prev)
                    else:
                        id_str += "{0},".format(prev)
                    cont_cnt = 0
                    prev = id
                    begin = id
        # append the last entry
        if cont_cnt > 0:
            id_str += "{0}-{1}".format(begin, prev)
        else:
            id_str += "{0}".format(prev)

        return id_str

    @staticmethod
    def is_duplicate_result(result1, result2):
        """
        Check whether two results are idenditical or not
        """
        return (
            result1["name"] == result2["name"]
            and result1["passed"] == result2["passed"]
            and result1["status"] == result2["status"]
            and (
                (result1["details"] == result2["details"])
                or result1["status"] in ["OK", "UNSUPPORTED"]
            )
        )

    @staticmethod
    def group_results_by_name(eval_result_details):
        """
        Group results by checkname and link node lnns to avoid duplicates
        """
        results_by_checks = {}
        for result in eval_result_details:
            name = result["name"]
            if name in results_by_checks:
                if not result["node"] or result["node"] == "cluster":
                    continue
                v = results_by_checks[name]
                dup = False
                for item in v:
                    if HealthCheckEvaluation.is_duplicate_result(item[0], result):
                        item[1].append(int(result["node"]))
                        dup = True
                        break
                if not dup:
                    v.append((result, [int(result["node"])]))

            else:
                nodes = []
                if result["node"] != "cluster" and result["node"]:
                    nodes.append(int(result["node"]))
                results = [(result, nodes)]
                results_by_checks[name] = results
        return results_by_checks

    def show_eval_result(self, eval_result):
        """
        Show HealthCheck evaluation results to console
        """
        self.show_eval_summary_stats(eval_result)
        eval_result_details = eval_result["details"]
        results_by_check = self.group_results_by_name(eval_result_details)
        for k, v in sorted(results_by_check.items()):
            for item in v:
                self.show_check_result(
                    item[0], include_node=True, node_ids=self.format_node_lnns(item[1])
                )
        logger.info("")  # add a newline

    def show_overall_summary_stats(self):
        """
        Show combined evaluation results
        """
        fail_count = 0
        total_count = 0
        us_cnt = 0
        for k, v in self.eval_results.items():
            if not v:
                continue
            check_results = v["details"]
            for result_item in check_results:
                total_count += 1
                if not result_item["passed"]:
                    fail_count += 1
                elif result_item["status"] == STATUS_UNS:
                    us_cnt += 1

        if fail_count > 0:
            logger.info(
                (
                    "\nOverall evaluation summary: Total(%d) %s(%d), "
                    + "Unsupported(%d), %s(%d)"
                ),
                total_count,
                display_success(MSG_PASSED),
                total_count - fail_count - us_cnt,
                us_cnt,
                display_failure(MSG_FAILED),
                fail_count,
            )
        else:
            logger.info(
                (
                    "\nOverall evaluation summary: Total(%d) %s(%d), "
                    + "Unsupported(%d)"
                ),
                total_count,
                display_success(MSG_PASSED),
                total_count - fail_count - us_cnt,
                us_cnt,
            )

    @staticmethod
    def sort_by_node_and_check_name(result_item):
        """
        Sort items by node lnn and check name
        """
        if not result_item["node"] or result_item["node"] == "cluster":
            return result_item["name"]
        else:
            return "%03d %s" % (int(result_item["node"]), result_item["name"])

    def show_results_by_node(self, results_by_node):
        """
        show HealthCheck evaluation results grouped by node/cluster
        """
        keys = list(results_by_node.keys())
        keys.sort()
        for k in keys:
            v = results_by_node[k]
            first_item = True
            if not (self.show_passed or self.show_unsupported):
                fail_count = 0
                for item in v:
                    if not item["passed"]:
                        fail_count += 1
                        break
                if fail_count == 0:
                    continue

            for item in v:
                if first_item:
                    first_item = False
                    logger.info("")
                    if k == "cluster":
                        logger.info(display_bold("Cluster:"))
                    elif k == "":
                        logger.info(display_bold("Off Cluster Unsupported:"))
                    else:
                        node_txt = "Node {0} (lnn):".format(int(k))
                        logger.info(display_bold(node_txt))
                    logger.info("")
                self.show_check_result(item, include_node=False)

    def show_all_eval_results(self):
        """
        Show all evaluation results
        """
        if self.group_by_nodes:
            # flatten the results and group them by nodes/cluster
            # for consolidated view
            results_by_node = self.group_results_by_node()
            self.show_results_by_node(results_by_node)
        else:
            for k, v in self.eval_results.items():
                if v:
                    self.show_eval_result(v)
        self.show_overall_summary_stats()

    def gather_evaluation_results(self, eval_requests, timeout=1800):
        """
        Monitor newly submitted or previously completed evaluations
        :param eval_requests: eval requests ({checkname:ID}) to be traced
        :param timeout: number of seconds to wait for completed results
        :return evaluation results {checkname: result}
        """
        wait_till = timeout + time.time()
        completed_checks = {}
        failed_checks = {}
        logger.debug("\nRunning evaluations {0}".format(eval_requests))
        while time.time() < wait_till:
            for check in eval_requests:
                eval_id = eval_requests[check]
                if check in completed_checks or check in failed_checks:
                    # this check has been completed
                    continue

                if IS_POWERSCALE:
                    uri = PAPI_HC_EVALUATIONS + eval_id
                    args = {"detailed": True}
                    response = basepapi.get(uri, args=args)
                    if response.status == 200:
                        evaluation = response.body.get("evaluations")[0]
                        if evaluation["run_status"] == "COMPLETED":
                            completed_checks[check] = evaluation
                            logger.debug(
                                "Completed %d out of %d evaluations",
                                len(completed_checks),
                                len(eval_requests),
                            )
                            continue
                    elif response.status == 404:
                        print("\n")
                        err_msg = (
                            "Specified evaluation (reference: '{0}') not found".format(
                                eval_id
                            )
                        )
                        logger.error(display_failure(err_msg))
                        failed_checks[check] = None
                else:
                    params = {"eval_id": eval_id, "checklist": check}
                    response = requests.get(OFHC_API + "results", params=params)
                    if response.status_code == 200:
                        evaluation = response.json()
                        if not evaluation:
                            continue

                        if evaluation["run_status"].upper() == "COMPLETED":
                            completed_checks[check] = evaluation
                            logger.debug(
                                "Completed %d out of %d evaluations",
                                len(completed_checks),
                                len(eval_requests),
                            )
                            continue
                        if evaluation["run_status"].upper() == "ABORTED":
                            print("\n")
                            err_msg = "Specified evaluation (reference: '{0}') not found".format(
                                check + eval_id
                            )
                            logger.error(display_failure(err_msg))
                            failed_checks[check] = None
                            continue
                    elif response.status_code == 404:
                        print("\n")
                        err_msg = (
                            "Specified evaluation (reference: '{0}') not found".format(
                                check + eval_id
                            )
                        )
                        logger.error(display_failure(err_msg))
                        failed_checks[check] = None
            else:
                if (len(completed_checks) + len(failed_checks)) == len(eval_requests):
                    print("\n")
                    break
                else:
                    print(".", end="")
                    sys.stdout.flush()
                    time.sleep(3)

        self.eval_results = completed_checks

    def match_eval_id_logset(self, eval_id, check, logset):
        params = {"checklist": check, "logset": logset}
        response = requests.get(OFHC_API + "evaluations", params=params)
        if response.status_code == 200:
            evaluations = response.json()
            if eval_id in evaluations:
                return True
        return False

    def populate_eval_id_list(self, logset):
        """
        Construct eval ID list to be queried for results
        """
        eval_requests = {}
        query_evals = []
        queried_evals = {}
        for eval in self.eval_names:
            matches = re.search(r"\d", eval)
            if matches:
                check_name = eval[: matches.start()]
                if IS_POWERSCALE:
                    eval_requests[check_name] = eval
                else:
                    eval_id = eval[matches.start() :]
                    # match eval id with logset, if not matching give warning
                    if not self.match_eval_id_logset(eval_id, check_name, logset):
                        err_msg = "Evaluation id '%s' not found for logset '%s'" % (
                            eval,
                            logset,
                        )
                        logger.error(display_failure(err_msg))
                    else:
                        eval_requests[check_name] = eval_id
            else:
                # populate list for which evalIDs to be queried
                query_evals.append(eval)
        if len(query_evals) > 0:
            # Find latest evaluation IDs for the checknames
            queried_evals = query_evaluation_ids_by_checknames(query_evals, logset)

        # combine the list
        combined_evals = eval_requests.copy()
        combined_evals.update(queried_evals)
        return combined_evals

    def get_system_info_as_html(self, logset):
        """
        Get Syststem information as html
        """
        system_info = get_system_info(logset)
        ret_str = ""
        if system_info:
            ret_str = "\n<div>\n"
            ret_str += "<h2>System Info</h2>\n"
            ret_str += '<table style="background-color:#C0C0C0">\n'

            ret_str += "<tr><td >Cluster Name</td><td>{0}</td></tr>\n".format(
                system_info["name"]
            )
            ret_str += "<tr><td>GUID</td><td>{0}</td></tr>\n".format(
                system_info["guid"]
            )
            ret_str += "<tr><td>OneFS Version</td><td>{0}</td></tr>\n".format(
                system_info["onefs_version"]
            )
            if "node_status" in system_info:
                node_status = system_info["node_status"]
                if node_status:
                    up_count = node_status["up_cnt"]
                    down_count = node_status["down_cnt"]
                    node_cnt_msg = ""
                    if node_status["down_cnt"] > 0:
                        down_lnns = node_status["down_lnns"]
                        down_nodes = ",".join([str(i) for i in down_lnns])
                        node_cnt_msg = (
                            "Total {0}, Up {1}, Down {2} " + "(lnn: {3})"
                        ).format(
                            up_count + down_count, up_count, down_count, down_nodes
                        )
                    else:
                        node_cnt_msg = "Total {0}, Up {1}".format(
                            up_count + down_count, up_count
                        )
                    ret_str += ("<tr><td>Node status</td><td>{0}</td></tr>\n").format(
                        node_cnt_msg
                    )

            ret_str += "<tr><td>Patches</td>"
            patch_detail_str = ""
            for patch in system_info["patches"]:
                patch_detail_str += "{0} -- {1} <br />".format(
                    patch["id"], patch["status"]
                )
            ret_str += "<td>{0}</td></tr>".format(patch_detail_str)
            ret_str += "</table>\n"
            ret_str += "</div>\n"
            return ret_str

    def get_overall_summary_as_html(self):
        """
        Get result summary as html
        """
        fail_count = 0
        total_count = 0
        uns_cnt = 0
        for k, v in self.eval_results.items():
            if not v:
                continue
            check_results = v["details"]
            for result_item in check_results:
                total_count += 1
                if not result_item["passed"]:
                    fail_count += 1
                elif result_item["status"] == STATUS_UNS:
                    uns_cnt += 1
        pass_count = total_count - fail_count - uns_cnt

        ret_str = "\n<div>\n"
        ret_str += "<h2>Overall HealthCheck Summary Results</h2>\n"
        ret_str += '<table style="background-color:#C0C0C0">\n'

        ret_str += (
            '<tr style="background-color:#00FF00"><td >Passed</td>'
            + "<td>{0}</td></tr>\n".format(pass_count)
        )

        ret_str += (
            '<tr style="background-color:#FF0000"><td>Failed</td>'
            + "<td>{0}</td></tr>\n".format(fail_count)
        )
        ret_str += (
            '<tr style="background-color:#C0C0C0"><td>Unsupported'
            + "</td><td>{0}</td></tr>\n".format(uns_cnt)
        )
        ret_str += (
            '<tr style="background-color:#FFFFFF"><td><b>Total</b>'
            + "</td><td>{0}</td></tr>\n".format(total_count)
        )
        ret_str += "</table>\n"
        ret_str += "</div>\n"
        return ret_str

    def get_indiv_summary_as_html(self, eval_result):
        """
        Get HealthCheck evaluation summary result as HTML string
        """
        fail_count = 0
        total_count = 0
        uns_cnt = 0
        check_results = eval_result["details"]
        for result_item in check_results:
            total_count += 1
            if not result_item["passed"]:
                fail_count += 1
            elif result_item["status"] == STATUS_UNS:
                uns_cnt += 1
        pass_count = total_count - fail_count - uns_cnt
        check_name = eval_result["checklist_id"]
        evaltime = time.strftime("%x %X", time.localtime(eval_result["start_time"]))

        ret_str = "\n<div>\n"
        ret_str += (
            "<h2>Summary result for checklist '{0}' (Timestamp: {1})</h2>\n".format(
                check_name, evaltime
            )
        )
        ret_str += (
            "<label><b>Total:{0}, Passed:{1}, Failed:{2}, "
            + "Un-supported:{3}</b></label><br/>"
        ).format(total_count, pass_count, fail_count, uns_cnt)
        ret_str += "</div>\n"
        return ret_str

    def get_detail_result_as_html_groupby_evals(self):
        """
        Get HealthCheck evaluation summary result grouped by node as
        HTML string
        """
        ret_str = ""
        for k, v in self.eval_results.items():
            if v:
                check_results = v["details"]
                if len(check_results) > 0:
                    ret_str += self.get_indiv_summary_as_html(v)

                    if not (self.show_passed or self.show_unsupported):
                        fail_count = 0
                        for item in check_results:
                            if not item["passed"]:
                                fail_count += 1
                                break
                        if fail_count == 0:
                            continue

                    ret_str += '<table style="background-color:#C0C0C0">'
                    ret_str += (
                        '<tr style="background-color:#C0C0C0">' + "<th>Check Name</th>"
                    )
                    ret_str += (
                        "<th>Node lnn</th><th>Status</th>" + "<th>Details</th></tr>"
                    )

                    for result in check_results:
                        if not self.is_item_displayable(result):
                            continue

                        bg_color = get_color_coded_status(result)[1]
                        ret_str += "<tr>"
                        ret_str += "<td>{0}</td>".format(result["name"])
                        ret_str += "<td>{0}</td>".format(result["node"])

                        msg = "PASSED" if result["passed"] else "FAILED"
                        if result["passed"] and result["status"] == "OK":
                            ret_str += (
                                '<td style="background-color:{0}">{1}</td>'.format(
                                    bg_color, msg
                                )
                            )
                        else:
                            ret_str += (
                                '<td style="background-color:{0}">' + "{1}({2})</td>"
                            ).format(bg_color, msg, result["status"])

                        detail_str = ""
                        if (not result["passed"]) or (
                            result["passed"] and result["status"] == "INFO"
                        ):
                            fail_details = result["details"]
                            if isinstance(fail_details, (str)):
                                fail_details = [fail_details]
                            elif isinstance(fail_details, (dict)):
                                fail_details = [
                                    "%s: %s" % (k, v)
                                    for (k, v) in list(fail_details.items())
                                ]

                            # Its array of details
                            for detail in fail_details:
                                if isinstance(detail, str):
                                    detail_str += detail
                                    detail_str += "<br />"
                                else:
                                    lines = json.dumps(detail, indent=2).split("\n")
                                    for line in lines:
                                        detail_str += line
                                        detail_str += "<br />"

                        ret_str += "<td>{0}</td>".format(detail_str)
                        ret_str += "</tr>"
                    ret_str += "</table>"
        return ret_str

    def group_results_by_node(self):
        """
        Get the HealthCheck results grouped by node
        """
        sorted_results = []
        evaluations = self.eval_results.values()
        # aggregate all results
        for evaluation in evaluations:
            if evaluation:
                sorted_results += evaluation["details"]

        # sort them by node + checkname
        sorted_results.sort(key=self.sort_by_node_and_check_name)

        # put the results into a dictionary with key as node lnn or cluster
        results_by_node = {}
        for item in sorted_results:
            node = item["node"]
            if node and node != "cluster":
                node = "%3d" % int(node)
            if node in results_by_node:
                # check for duplicates
                cur_items = results_by_node[node]
                dup_item = False
                for check in cur_items:
                    if check["name"] == item["name"]:
                        dup_item = True
                        break
                if not dup_item:
                    results_by_node[node].append(item)
            else:
                results_by_node[node] = [item]
        return results_by_node

    def get_detail_result_as_html_groupby_nodes(self):
        """
        Get HealthCheck results grouped by nodes and formatted as HTML
        """
        results_by_node = self.group_results_by_node()
        ret_str = ""
        keys = list(results_by_node.keys())
        keys.sort()
        ret_str += "<h2>Summary results grouped by nodes </h2>"

        ret_str += '<table style="background-color:#C0C0C0;">'
        ret_str += "<tr><th>Node lnn</th><th>Details</th></tr>"
        for k in keys:
            v = results_by_node[k]
            if not (self.show_passed or self.show_unsupported):
                fail_count = 0
                for item in v:
                    if not item["passed"]:
                        fail_count += 1
                        break
                if fail_count == 0:
                    continue

            node_txt = ""
            if k == "cluster":
                node_txt = "Cluster"
            elif k == "":
                node_txt = "Off Cluster Unsupported"
            else:
                node_txt = "{0}".format(int(k))

            ret_str += (
                '<tr><td style="vertical-align: center;'
                + 'text-align:center;">{0}</td>'.format(node_txt)
            )
            ret_str += '<td><table style="vertical-align: center;">'
            if len(v) > 0:
                for result in v:
                    if result["passed"] and not (
                        self.show_passed or self.show_unsupported
                    ):
                        continue
                    if result["status"] == STATUS_UNS and not self.show_unsupported:
                        continue

                    bg_color = get_color_coded_status(result)[1]
                    ret_str += "<tr>"
                    ret_str += "<td>{0}</td>".format(result["name"])

                    msg = "PASSED" if result["passed"] else "FAILED"
                    if result["passed"] and result["status"] == "OK":
                        ret_str += (
                            '<td style="background-color:{0}; '
                            + "vertical-align:center; text-align:center;"
                            + '">{1}</td>'
                        ).format(bg_color, msg)
                    else:
                        ret_str += (
                            '<td style="background-color:{0}; '
                            + "vertical-align:center; text-align:center;"
                            + '">{1}({2})</td>'
                        ).format(bg_color, msg, result["status"])

                    detail_str = ""
                    if (not result["passed"]) or (
                        result["passed"] and result["status"] == "INFO"
                    ):
                        fail_details = result["details"]
                        if isinstance(fail_details, (str)):
                            fail_details = [fail_details]
                        elif isinstance(fail_details, (dict)):
                            fail_details = [
                                "%s: %s" % (k, v)
                                for (k, v) in list(fail_details.items())
                            ]

                        # Its array of details
                        for detail in fail_details:
                            if isinstance(detail, str):
                                detail_str += detail
                                detail_str += "<br />"
                            else:
                                lines = json.dumps(detail, indent=2).split("\n")
                                for line in lines:
                                    detail_str += line
                                    detail_str += "<br />"

                    ret_str += "<td>{0}</td>".format(detail_str)
                    ret_str += "</tr>"
                ret_str += "</table></td></tr>"
        ret_str += "</table>"
        return ret_str

    def get_detail_results_as_html(self):
        """
        Get HealthCheck results formatted as html output
        """
        if self.group_by_nodes:
            return self.get_detail_result_as_html_groupby_nodes()
        else:
            return self.get_detail_result_as_html_groupby_evals()

    def get_html_header(self):
        """
        Get header part of the HTML output
        """
        ret_str = ""
        ret_str += (
            "<html>\n<head>\n<title> Health Check Evaluation " + "Results\n</title>\n"
        )
        ret_str += "<style>\ntable, th, td {\n"
        ret_str += "border: 1px solid black;\n border-collapse: collapse;}\n"
        ret_str += "th, td {\n  padding: 10px;}\n"
        ret_str += "</style></head>"
        ret_str += "<body>\n"
        return ret_str

    def get_html_footer(self):
        """
        Get HTML footer
        """
        return "\n</body>\n</html>\n"

    def get_html_output(self, logset):
        """
        Top level function to get HTML output
        """
        ret_str = self.get_html_header()

        ret_str += self.get_system_info_as_html(logset)
        ret_str += self.get_overall_summary_as_html()
        ret_str += self.get_detail_results_as_html()

        ret_str += self.get_html_footer()
        return ret_str

    def save_result_to_file(self, logset):
        """
        Save the HealthCheck result as html
        """
        if not self.filepath:
            return
        if not os.path.exists(self.filepath) or not os.path.isdir(self.filepath):
            err_msg = "Unable to save results in file, specified path does not exist"
            logger.error(display_failure(err_msg))
            return
        if not self.filepath.endswith(os.path.sep):
            self.filepath += os.path.sep

        filename = self.filepath + "healthcheck_results_{}.html".format(
            datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S")
        )
        try:
            logger.info("HTML output saved in %s", filename)
            with open(filename, "w") as out:
                out.write(self.get_html_output(logset))
        except (IOError) as error:
            err_msg = "Failed to save results as html file: error: {e}".format(e=error)
            logger.error(display_failure(err_msg))

    def send_email(self, logset):
        """
        Send healthcheck results to specified email addresses
        """
        if IS_POWERSCALE:
            if not self.email_addresses:
                return
            try:
                response = basepapi.get(PAPI_CLUSTER_EMAIL)
                email_settings = None
                if response.status == 200:
                    email_settings = response.body["settings"]
                if not email_settings:
                    err_msg = "Email send failed: SMTP not setup"
                    logger.error(display_failure(err_msg))
                    return

                from_address = None
                if email_settings:
                    relay_addr = str(email_settings["mail_relay"])
                    if not relay_addr or len(relay_addr.strip()) == 0:
                        err_msg = "Email send failed: SMTP server not configured"
                        logger.error(display_failure(err_msg))
                        return
                    from_address = str(email_settings["mail_sender"])
                if not from_address:
                    from_address = EMAIL_FROM_ADDRESS

                Emailer.send_email(
                    self.email_addresses,
                    self.get_html_output(logset),
                    from_address,
                    EMAIL_SUBJECT,
                    charset="utf-8",
                )
                logger.info("Email sent with the HealthCheck results")

            except Exception as error:
                err_msg = "Failed to send email notifications, error: {e}".format(
                    e=error
                )
                logger.error(display_failure(err_msg))
        else:
            if not self.email_addresses:
                return
            try:
                sender_email = "isihealthcheckoffcluster@dell.com"
                email_info = MIMEMultipart()
                email_info["From"] = sender_email
                email_info["To"] = self.email_addresses
                email_info["Subject"] = EMAIL_SUBJECT
                body = self.get_html_output(logset)
                email_info.attach(MIMEText(body, "html"))
                body_contents = email_info.as_string()
                OFHC_SMTP = os.environ["OFHC_SMTP"]
                smtpObj = smtplib.SMTP(OFHC_SMTP)
                smtpObj.sendmail(sender_email, self.email_addresses, body_contents)
                logger.info("Email sent with the HealthCheck results")
                smtpObj.quit()
            except Exception as error:
                err_msg = "Failed to send email notifications, error: {e}".format(
                    e=error
                )
                logger.error(display_failure(err_msg))

    def post_process_results(self, logset):
        self.save_result_to_file(logset)
        self.send_email(logset)

    def view_evaluations(self, logset):
        """
        Retrive latest evaluation results and show results
        :return None
        """
        combined_evals = self.populate_eval_id_list(logset)
        self.gather_evaluation_results(combined_evals)
        # show results
        self.show_all_eval_results()
        self.post_process_results(logset)

    def run_evaluations(
        self, logset, override=False, priority=False, check_parameters=None
    ):
        """
        Run evaluations
        """
        eval_requests, bad_request = self.submit_evals(
            logset, override, priority, check_parameters
        )
        self.gather_evaluation_results(eval_requests)
        self.show_all_eval_results()
        self.post_process_results(logset)


def query_evaluation_ids_by_checknames(checknames, logset):
    """
    Query and retrive latest evaluation IDs for check/checklist names
    :param checknames: list of checklist/checkname to be searched for
    : return {checkname: latest-eval-ID}
    """
    if IS_POWERSCALE:
        uri = PAPI_HC_EVALUATIONS
        args = {"level": "list"}
        eval_requests = {}
        latest_eval_ids = {}

        logger.debug("Checking existing evaluations by check names")
        response = basepapi.get(uri, args=args)
        if response.status == 200:
            evaluations = response.body.get("evaluations")
            for eval in evaluations:
                eval_name = eval["checklist_id"]
                if eval_name in checknames:
                    if eval_name in eval_requests.keys():
                        eval_requests[eval_name].append(eval["id"])
                    else:
                        eval_requests[eval_name] = [eval["id"]]

            for item in checknames:
                if item not in eval_requests:
                    err_msg = (
                        "No matching previous evaluation results found for '%s'" % item
                    )
                    logger.error(display_failure(err_msg))

            for k, v in eval_requests.items():
                v.sort(reverse=True)
                if len(v) > 0:
                    latest_eval_ids[k] = v[0]
    else:
        latest_eval_ids = {}
        logger.debug("Checking existing evaluations by check names")
        for check in checknames:
            params = {"checklist": check, "logset": logset}
            response = requests.get(OFHC_API + "evaluations", params=params)
            if response.status_code == 200:
                if len(response.json()) > 0:
                    latest_eval_ids[check] = response.json()[0]

        for item in checknames:
            if item not in latest_eval_ids:
                err_msg = (
                    "No matching previous evaluation results found for '%s'" % item
                )
                logger.error(display_failure(err_msg))

    return latest_eval_ids


def get_checklist():
    """
    Retrive list of HealthCheck checklist available in the system
    """

    logger.debug("Getting HealthCheck checklist")
    checklists = {}

    if IS_POWERSCALE:
        uri = PAPI_HC_CHKLIST
        response = basepapi.get(uri)
        if response.status == 200:
            checklists = response.body.get("checklists")
        else:
            logger.error("Error accessing checklists, response {0}".format(response))
        return checklists

    else:
        uri = OFHC_API + "checklists"

        response = requests.get(uri)

        if response.status_code == 200:
            checklists = response.json()
        else:
            logger.error("Error accessing checklists, response {0}".format(response))
        return checklists


def show_checklist(checklists):
    """
    Display healthCheck checklists in console
    """
    logger.info("\n{0:<25}     {1:<30}\n".format("Name", "Description"))
    for check in checklists:
        logger.info(
            "{0:<25}     {1:<30}".format(
                display_bold(check["id"]), check["description"]
            )
        )


def view_checklist():
    """
    Retrive and display list of checklist available in the system
    """
    checklists = get_checklist()
    show_checklist(checklists)


def show_checklists_by_domain():
    """
    Display healthCheck checklists grouped by domain
    """
    logger.info("\n{0:<20}  {1}\n".format("Domain Name", "Included CheckLists"))

    for k, v in sorted(CHECKS_BY_DOMAIN.items()):
        checklist_names = ",".join(v[1])
        logger.info("{0:<25}     {1:<30}".format(display_bold(v[0]), checklist_names))


def get_check_details():
    """
    Get list of all checks with details
    """
    logger.debug("Getting detail information for all checks")

    if IS_POWERSCALE:
        check_details = {}

        uri = PAPI_HC_ITEMS
        response = basepapi.get(uri)

        if response.status == 200:
            details = response.body.get("items")
            for check in details:
                check_details[check["id"]] = check["summary"]
        elif response.status == 404:
            err_msg = "Unable to gather check details from the system"
            logger.error(display_failure(err_msg))
        return check_details
    else:
        checks = {}
        uri = OFHC_API + "checks"

        response = requests.get(uri)

        if response.status_code == 200:
            checks = response.json()
        else:
            logger.error("Error accessing checklists, response {0}".format(response))
        return checks


def get_checks(checklist_name):
    """
    Get list of checks for specified checklist
    """
    logger.debug("Getting checks for checklist '%s'", checklist_name)

    if IS_POWERSCALE:
        checks = []
        uri = PAPI_HC_CHKLIST + checklist_name
        response = basepapi.get(uri)
        if response.status == 200:
            checklist = response.body.get("checklists")
            if len(checklist) == 1:
                for check in checklist[0]["items"]:
                    checks.append(check)
        elif response.status == 404:
            err_msg = "Specified checklist '%s' not found" % checklist_name
            logger.error(display_failure(err_msg))
        return checks
    else:
        checks = []
        uri = OFHC_API + "checklist"
        params = {"checklist": checklist_name}
        response = requests.get(uri, params=params)

        if response.status_code == 200:
            checks = response.json()
        else:
            logger.error("Error accessing checklists, response {0}".format(response))
        return checks


def show_checks(checklist_name, checks, check_details):
    """
    Show list of checks in console
    """
    logger.info(
        "\nFound {0} checks for checklist '{1}'\n".format(
            len(checks), display_bold(checklist_name)
        )
    )

    if IS_POWERSCALE:
        sorted_checks = sorted(checks, key=lambda x: x["name"])

        wrapper = textwrap.TextWrapper(width=70)

        for check in sorted_checks:
            check_summary = "N/A"
            check_name = check["name"]
            if check_name in check_details:
                check_summary = check_details[check_name]

            lines = wrapper.wrap(text=check_summary)
            header = True
            for line in lines:
                if header:
                    logger.info("{0:<30}     {1:<40}".format(check_name, line))
                    header = False
                else:
                    logger.info("{0:<30}     {1:<40}".format("", line))
    else:

        sorted_checks = sorted(checks)

        wrapper = textwrap.TextWrapper(width=70)

        for check_name in sorted_checks:
            check_summary = "N/A"
            if check_name == check_details[check_name]["name"]:
                check_summary = check_details[check_name]["summary"]

            lines = wrapper.wrap(text=check_summary)
            header = True
            for line in lines:
                if header:
                    logger.info("{0:<30}     {1:<40}".format(check_name, line))
                    header = False
                else:
                    logger.info("{0:<30}     {1:<40}".format("", line))


def view_checks(checklist_name):
    """
    Get list of checks for a checklist and display
    """
    checks = get_checks(checklist_name)
    if checks:
        check_details = get_check_details()
        show_checks(checklist_name, checks, check_details)
    else:
        print("Invalid checkname")
        return


"""
def user_login(args):
    credential = args.split(",")
    response = requests.get(OFHC_API + "login/" + credential[0] + "/" + credential[1])
    if response.cookies:
        print(response.cookies["Name"], "logged in successfully")
        with open("cookie", "wb") as f:
            pickle.dump(response.cookies, f)
    else:
        print(response.json())


def user_logout():
    response = requests.get(OFHC_API + "logout")
    print(response.json())
    os.remove("cookie")
"""


def show_historical_eval_results(historical_eval_results):
    """
    Show summary of historical healthcheck evaluation result
    """
    if not historical_eval_results:
        logger.error("No historical results available for specified checks")
        return
    for k, v in historical_eval_results.items():
        if isinstance(v, list):
            header = True
            for eval_result in reversed(v):
                HealthCheckEvaluation.show_eval_summary_stats(
                    eval_result, header=header, historical=True
                )
                header = False
        else:
            logger.warn(
                "Received invalid data type in eval results: key:"
                + "{0}, value: {1}".format(k, type(v))
            )
        logger.info("")


def get_historical_eval_results(eval_requests):
    """
    Retrieve results for previously completed evaluations
    param eval_requests: a dictionary with checkname and previous eval IDs
                        {checkname: [eval-ids]}
    return evaluation results grouped by checkname {checkname: [results]}
    """
    detail_eval_results = {}
    logger.debug("\nChecking historical evaluation results {0}".format(eval_requests))

    for check in eval_requests:
        # get the list of eval IDs for the specified check
        eval_ids = eval_requests[check]
        eval_results = []
        for eval_id in eval_ids:
            uri = PAPI_HC_EVALUATIONS + eval_id
            args = {"detailed": True}
            response = basepapi.get(uri, args=args)
            if response.status == 200:
                evaluation = response.body.get("evaluations")[0]
                if evaluation["run_status"] == "COMPLETED":
                    eval_results.append(evaluation)
                    continue
                elif response.status == 404:
                    err_msg = (
                        "Specified evaluation (reference: '{0}') not found"
                    ).format(eval_id)
                    logger.error(display_failure(err_msg))

        if len(eval_results) > 0:
            detail_eval_results[check] = eval_results

    return detail_eval_results


def get_historical_evallist(checknames, limit=None):
    """
    Get list of historical evaluations for the specified checknames
    :param checknames: list of checkname to be searched
    :param limit: number of items to return
    :return a dictionary with {checkname: [list of evaluation IDs]}
    """
    if not checknames:
        logger.error(
            (
                "Checklist name(s) are not specified, see option '--checklist'"
                + " for details"
            )
        )
        os._exit(1)

    checkers = []
    if isinstance(checknames, str):
        checkers = checknames.strip().split(",")

    if len(checkers) == 0:
        err_msg = (
            "Checklist names are not specified, see option '--checklist' "
            + "for details"
        )
        logger.error(display_failure(err_msg))
        os._exit(1)

    max_item_count = limit if limit else 3
    logger.debug("Getting historical evaluation list for checks '%s'", checknames)

    all_eval_requests = {}
    filtered_eval_requests = {}

    uri = PAPI_HC_EVALUATIONS
    args = {"level": "list"}

    response = basepapi.get(uri, args=args)
    if response.status == 200:
        evaluations = response.body.get("evaluations")
        for eval in evaluations:
            eval_name = eval["checklist_id"]
            if eval_name in checkers:
                if eval_name in all_eval_requests.keys():
                    all_eval_requests[eval_name].append(eval["id"])
                else:
                    all_eval_requests[eval_name] = [eval["id"]]

    for item in checkers:
        if item not in all_eval_requests:
            err_msg = "No matching previous evaluation found for '{0}'".format(item)
            logger.error(display_failure(err_msg))

    for k, v in all_eval_requests.items():
        v.sort(reverse=True)
        if len(v) > max_item_count:
            v = v[:max_item_count]
        filtered_eval_requests[k] = v

    return filtered_eval_requests


def view_historical(checknames, limit=None):
    """
    Retrive and show historical healthcheck evaluation results
    :param checknames: list of checkname to be searched
    :param limit: number of items to return
    :return None
    """
    historical_eval_ids = get_historical_evallist(checknames, limit)
    logger.info("Retrieving historical health check results, please wait...")
    historical_results = get_historical_eval_results(historical_eval_ids)
    show_historical_eval_results(historical_results)


def show_system_info(logset):
    """
    Show system information
    """
    system_info = get_system_info(logset)
    if system_info:
        logger.info("{0:<30}     {1}".format("Cluster Name:", system_info["name"]))
        logger.info("{0:<30}     {1}".format("Cluster GUID:", system_info["guid"]))
        logger.info(
            "{0:<30}     {1}".format("OneFS version:", system_info["onefs_version"])
        )

        # show node status
        if "node_status" in system_info:
            node_status = system_info["node_status"]
            if node_status:
                up_count = node_status["up_cnt"]
                down_count = node_status["down_cnt"]
                if node_status["down_cnt"] > 0:
                    down_lnns = node_status["down_lnns"]
                    down_nodes = ",".join([str(i) for i in down_lnns])
                    node_cnt_msg = "Total {0}, Up {1}, Down {2} (lnn: {3})".format(
                        up_count + down_count, up_count, down_count, down_nodes
                    )
                    logger.info(
                        "{0:<30}     {1}".format(
                            "Node status:", display_failure(node_cnt_msg)
                        )
                    )
                else:
                    node_cnt_msg = "Total {0}, Up {1}".format(
                        up_count + down_count, up_count
                    )
                    logger.info(
                        "{0:<30}     {1}".format(
                            "Node status:", display_success(node_cnt_msg)
                        )
                    )

        # show patch details
        logger.info("System patches:")
        if "patches" in system_info:
            for patch in system_info["patches"]:
                logger.info("    {0:<40}     {1}".format(patch["id"], patch["status"]))


def get_system_info(logset):
    """
    Gather system info (version, patches etc)
    """
    logger.debug("Getting system information")
    system_info = {}

    if not IS_POWERSCALE:
        logsetlist = []
        logsetlist = logset.split(",")

        for logset in logsetlist:
            params = {"logset": logset}
            response = requests.get(OFHC_API + "system", params=params)
            if response.status_code == 200:
                system_info = response.json()
                up_count = 0
                down_count = 0
                down_lnns = []
                if not system_info:
                    continue
                for node in system_info["devices"]:
                    if node["is_up"]:
                        up_count += 1
                    else:
                        down_count += 1
                        down_lnns.append(node["lnn"])
                    system_info["onefs_version"] = node["version"]

                system_info["node_status"] = {
                    "up_cnt": up_count,
                    "down_cnt": down_count,
                    "down_lnns": down_lnns,
                }

            else:
                err_msg = "Error accessing cluster configuration, response {0}".format(
                    response
                )
                logger.error(display_failure(err_msg))

        return system_info

    config_uri = PAPI_CLUSTER_CONFIG
    response = basepapi.get(config_uri)
    if response.status == 200 and response.body:
        name = response.body.get("name")
        system_info["name"] = name

        guid = response.body.get("guid")
        system_info["guid"] = guid

        # get the release version info
        version = response.body.get("onefs_version")
        system_info["onefs_version"] = version["release"]

        # get the node details
        up_count = 0
        down_count = 0
        nodes = response.body.get("devices")
        down_lnns = []
        for node in nodes:
            if node["is_up"]:
                up_count += 1
            else:
                down_count += 1
                down_lnns.append(node["lnn"])

        system_info["node_status"] = {
            "up_cnt": up_count,
            "down_cnt": down_count,
            "down_lnns": down_lnns,
        }

    else:
        err_msg = "Error accessing cluster configuration, response {0}".format(response)
        logger.error(display_failure(err_msg))

    logger.debug("Getting system patch information")
    upgrade_uri = PAPI_UPGRADE_PATCHES
    response = basepapi.get(upgrade_uri)
    if response.status == 200 and response.body:
        patches = response.body.get("patches")
        system_info["patches"] = patches
    else:
        err_msg = "Error accessing patch details, response {0}".format(response)
        logger.error(display_failure(err_msg))

    return system_info


def run_evaluations_by_domain(args):
    """
    Run HealthCheck results by Domain
    """
    if IS_POWERSCALE:
        logger.debug("Running HealthCheck evaluations by domain")
        logset = ""
        override = False
        priority = False
        check_parameters = None
        show_system_info(logset)
        checklist_domain = args.run_by_domain
        if checklist_domain is None or len(checklist_domain) == 0:
            logger.error(display_failure("HealthCheck Domain name not specified"))
            return
        key = str(checklist_domain).lower().strip()
        if key in CHECKS_BY_DOMAIN:
            checknames = ",".join(CHECKS_BY_DOMAIN[key][1])
            logger.debug(checknames)
            hce = HealthCheckEvaluation(
                checknames,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset,
                args.filepath,
                args.send_email,
            )
            hce.run_evaluations(logset, override, priority, check_parameters)
        else:
            err_msg = (
                "Specified HealthCheck domain '{0}' not found. "
                + "Please use --checklist_by_domain command to see available list"
            )
            logger.error(display_failure(err_msg.format(checklist_domain)))
        logger.debug("HealthCheck evaluations run by domain completed")

    else:
        override = args.override
        priority = args.priority
        check_parameters = args.check_parameters
        checklist_domain = args.run_by_domain
        if checklist_domain is None or len(checklist_domain) == 0:
            logger.error(display_failure("HealthCheck Domain name not specified"))
            return
        key = str(checklist_domain).lower().strip()
        if key in CHECKS_BY_DOMAIN:
            checknames = ",".join(CHECKS_BY_DOMAIN[key][1])
            args.checknames = checknames
            logger.debug(checknames)
            logsetlist_process(args, "run_by_domain")
        else:
            err_msg = (
                "Specified HealthCheck domain '{0}' not found. "
                + "Please use --checklist_by_domain command to see available list"
            )
            logger.error(display_failure(err_msg.format(checklist_domain)))
        logger.debug("HealthCheck evaluations run by domain completed")


def view_evaluations_by_domain(args):
    """
    View HealthCheck results by Domain
    """
    if IS_POWERSCALE:
        logger.debug("View HealthCheck evaluations by domain")
        logset = ""
        show_system_info(logset)
        checklist_domain = args.view_by_domain
        if checklist_domain is None or len(checklist_domain) == 0:
            logger.error(display_failure("HealthCheck Domain name not specified"))
            return
        key = str(checklist_domain).lower().strip()
        if key in CHECKS_BY_DOMAIN:
            checknames = ",".join(CHECKS_BY_DOMAIN[key][1])
            hce = HealthCheckEvaluation(
                checknames,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset,
                args.filepath,
                args.send_email,
            )
            hce.view_evaluations(logset)
        else:
            err_msg = (
                "Specified HealthCheck domain '{0}' not found. "
                + "Please use --checklist_by_domain command to see available Domains"
            )
            logger.error(display_failure(err_msg.format(checklist_domain)))
        logger.debug("Viewing HealthCheck evaluations run by domain completed")
    else:
        checklist_domain = args.view_by_domain
        if checklist_domain is None or len(checklist_domain) == 0:
            logger.error(display_failure("HealthCheck Domain name not specified"))
            return
        key = str(checklist_domain).lower().strip()
        if key in CHECKS_BY_DOMAIN:
            checknames = ",".join(CHECKS_BY_DOMAIN[key][1])
            args.checknames = checknames
            logsetlist_process(args, "view_by_domain")
        else:
            err_msg = (
                "Specified HealthCheck domain '{0}' not found. "
                + "Please use --checklist_by_domain command to see available Domains"
            )
            logger.error(display_failure(err_msg.format(checklist_domain)))


def logset_view_evaluations(hce):
    combined_evals = hce.populate_eval_id_list(hce.logset)
    hce.gather_evaluation_results(combined_evals)
    return hce


def logset_run_evaluations(hce, override, priority, check_parameters):
    eval_requests, bad_request = hce.submit_evals(
        hce.logset, override, priority, check_parameters
    )
    if bad_request:
        hce.bad_request = bad_request
        return hce
    hce.gather_evaluation_results(eval_requests)
    return hce


def logset_view_by_domain_evaluations(hce):
    combined_evals = hce.populate_eval_id_list(hce.logset)
    hce.gather_evaluation_results(combined_evals)
    return hce


def logset_run_by_domain_evaluations(hce, override, priority, check_parameters):
    eval_requests, bad_request = hce.submit_evals(
        hce.logset, override, priority, check_parameters
    )
    if bad_request:
        hce.bad_request = bad_request
        return hce
    hce.gather_evaluation_results(eval_requests)
    return hce


def logsetlist_process(args, task):
    logset = args.logset if args.logset else ""
    logsetlist = []
    logsetlist = logset.split(",")
    hce_objectlist = []
    for logset_instance in logsetlist:
        if task == "view":
            hce = HealthCheckEvaluation(
                args.view,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset_instance,
                args.filepath,
                args.send_email,
            )
        if task == "view_by_domain":
            hce = HealthCheckEvaluation(
                args.checknames,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset_instance,
                args.filepath,
                args.send_email,
            )
        if task == "run":
            hce = HealthCheckEvaluation(
                args.run,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset_instance,
                args.filepath,
                args.send_email,
            )
        if task == "run_by_domain":
            hce = HealthCheckEvaluation(
                args.checknames,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset_instance,
                args.filepath,
                args.send_email,
            )

        hce_objectlist.append(hce)

    with concurrent.futures.ThreadPoolExecutor(max_workers=LOGSET_WORKERS) as executor:
        if task == "run" or task == "run_by_domain":
            future_task = {
                executor.submit(
                    eval("logset_" + task + "_evaluations"),
                    obj_hce,
                    args.override,
                    args.priority,
                    args.check_parameters,
                ): obj_hce
                for obj_hce in hce_objectlist
            }
        else:
            future_task = {
                executor.submit(
                    eval("logset_" + task + "_evaluations"), obj_hce
                ): obj_hce
                for obj_hce in hce_objectlist
            }
        for future in concurrent.futures.as_completed(future_task):
            obj_hce = future.result()
            if not obj_hce.bad_request:
                print("Logset " + obj_hce.logset)
                show_system_info(obj_hce.logset)
                obj_hce.show_all_eval_results()
                obj_hce.post_process_results(obj_hce.logset)
        wait(future_task)


def check_smtp_env_variable(args):
    if IS_POWERSCALE:
        return
    if not args.send_email:
        return
    if "OFHC_SMTP" not in os.environ:
        logger.error(
            display_failure(
                "Please set OFHC_SMTP environment variable to use --send-email"
            )
        )
        os._exit(1)


def run(args):
    """formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    )
    """
    formatter = logging.Formatter("%(message)s")
    console_log = logging.StreamHandler()
    console_log.setFormatter(formatter)

    if args.verbose:
        logger.setLevel(logging.DEBUG)
    logger.addHandler(console_log)

    begin_time = datetime.datetime.now()

    logger.debug("Evaluation started at {0}".format(datetime.datetime.now()))
    # parse tthe command and options and execute
    if args.view:
        if IS_POWERSCALE:
            logset = ""
            show_system_info(logset)
            hce = HealthCheckEvaluation(
                args.view,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset,
                args.filepath,
                args.send_email,
            )
            hce.view_evaluations(logset)
        else:
            check_smtp_env_variable(args)
            logsetlist_process(args, "view")
    elif args.view_by_domain:
        check_smtp_env_variable(args)
        view_evaluations_by_domain(args)
    elif IS_POWERSCALE and args.historical:
        view_historical(args.historical, args.limit)
    elif args.checklist:
        view_checklist()
    elif args.checklist_by_domain:
        show_checklists_by_domain()
    elif args.checks:
        view_checks(args.checks)
    elif args.run:
        if IS_POWERSCALE:
            logset = ""
            show_system_info(logset)
            hce = HealthCheckEvaluation(
                args.run,
                args.show_passed,
                args.group_by_nodes,
                args.show_unsupported,
                logset,
                args.filepath,
                args.send_email,
            )
            hce.run_evaluations(logset)
        else:
            check_smtp_env_variable(args)
            logsetlist_process(args, "run")
    elif args.run_by_domain:
        check_smtp_env_variable(args)
        run_evaluations_by_domain(args)
    else:
        logger.info("No command specified, please use --help for options")

    end_time = datetime.datetime.now()
    elapsed = end_time - begin_time

    logger.info(
        "\nRequest completed in %s seconds",
        time.strftime("%H:%M:%S", time.gmtime(elapsed.total_seconds())),
    )


def add_historical_view_options(parser):
    historical_view_options = parser.add_argument_group("Historical view Options")
    historical_view_options.add_argument(
        "--limit",
        dest="limit",
        type=int,
        nargs="?",
        const=3,
        default=None,
        help="Specify max number of records to be displayed for historical "
        + "results, default is 3.\n"
        + "e.g --limit 5\n"
        + "Ignored for all commands except '--view_historical'\n\n",
    )


def add_offcluster_options(parser):
    offcluster_options = parser.add_argument_group("Off cluster Options")
    offcluster_options.add_argument(
        "--logset",
        type=str,
        default=None,
        help="Specify the logsets for off-cluster healthcheck evaluations",
    )
    offcluster_options.add_argument(
        "--override",
        action="store_true",
        default=False,
        help="Overrides the cached responses for offcluster evaluation.\n"
        + "Use with --run --logset, --run_by_domain --logset options.",
    )
    offcluster_options.add_argument(
        "--priority",
        action="store_true",
        default=False,
        help="Prioritizes the request for offcluster evaluation.\n"
        + "Use with --run --logset, --run_by_domain --logset options.",
    )


def usage():
    return """
        %(prog)s [Command] [Options]
        """


def main():
    """
    parse args, complete task
    """

    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawTextHelpFormatter,
        usage=usage(),
    )

    parser.add_argument("--version", action="version", version=TOOL_VERSION)

    group1 = parser.add_argument_group("Commands")
    group2 = parser.add_argument_group("Result view Options")

    group1.add_argument(
        "--run",
        type=str,
        nargs="?",
        const="basic",
        default=None,
        help="Command to start a new HealthCheck evaluation.\n"
        + "Specify comma separated list of check(s)/checklist(s) to be run.\n"
        + "Runs 'basic' scan when no values specified\n"
        + "Example: '--run basic,nfs,auth'\n"
        + "Use option '--checklist' to see list of available check lists.\n"
        + "By default result shows only failed checks.\n"
        + "Following options can be used in any combination\n"
        + "Use option '--show_passed' to see passed checks\n"
        + "Use option '--show_unsupported' to see unsupported checks\n"
        + "Use option '--group_by_nodes' to see results grouped by nodes\n"
        + "Use option '--filepath' <path> to save result as HTML\n"
        + "Use option '--send_email' <email addresses> to send results"
        + " in Email.\n\n",
    )

    group1.add_argument(
        "--run_by_domain",
        type=str,
        default=None,
        help="Command to start a new HealthCheck evaluation by domain.\n"
        + "Use option '--checklist_by_domain' to see available domain list.\n"
        + "Example: '--run_by_domain Network'\n"
        + "By default results show only failed checks.\n"
        + "Following options can be used in any combination\n"
        + "Use option '--show_passed' to include 'passed' checks in results\n"
        + "Use option '--show_unsupported' to include 'unsupported' checks\n"
        + "Use option '--group_by_nodes' to see results grouped by nodes\n"
        + "Use option '--filepath' <path> to save result as HTML\n"
        + "Use option '--send_email' <email addresses> to send results"
        + " in Email.\n\n",
    )

    group1.add_argument(
        "--view",
        type=str,
        nargs="?",
        const="basic",
        default=None,
        help="Command to view results from earlier HealthCheck evaluations.\n"
        + "Specify comma separated list of checklist name(s) or exact \n"
        + "evaluation ID(s)\n"
        + "Displays results for 'basic' when no values specified\n"
        + "Example: '--view basic,nfs20211122T2228'\n"
        + "Use option '--checklist' to see list of available check lists.\n"
        + "By default results show only failed checks.\n"
        + "Following options can be used in any combination\n"
        + "Use option '--show_passed' to include 'passed' checks in result\n"
        + "Use option '--show_unsupported' to include 'unsupported' checks\n"
        + "Use option '--group_by_nodes' to see results grouped by nodes\n"
        + "Use option '--filepath' <path> to save result as HTML\n"
        + "Use option '--send_email' <email addresses> to send results in "
        + "Email.\n\n",
    )

    group1.add_argument(
        "--view_by_domain",
        type=str,
        default=None,
        help="Command to view HealthCheck results by domain.\n"
        + "Use option '--checklist_by_domain' to see available domain list.\n"
        + "Example: '--view_by_domain FileSystem'\n"
        + "By default results show only failed checks.\n"
        + "Following options can be used in any combination\n"
        + "Use option '--show_passed' to include 'passed' checks in result\n"
        + "Use option '--show_unsupported' to include 'unsupported' checks\n"
        + "Use option '--group_by_nodes' to see results grouped by nodes\n"
        + "Use option '--filepath' <path> to save result as HTML\n"
        + "Use option '--send_email' <email addresses> to send results"
        + " in Email.\n\n",
    )
    if IS_POWERSCALE:
        group1.add_argument(
            "--view_historical",
            dest="historical",
            type=str,
            nargs="?",
            const="basic",
            default=None,
            help="Command to view historical HealthCheck summary results.\n"
            "Specify comma separated checklist name(s).\n"
            + "Displays results for 'basic' when no values specified\n"
            + "Use option '--limit' to see number of historical items\n"
            + "eg --view_historical basic,nfs,auth --limit 5\n"
            + "Use the ID from this output, along with '--view' command to see\n"
            + "additional details of the evaluation\n\n",
        )

    group1.add_argument(
        "--checklist",
        action="store_true",
        default=None,
        help="Command to see list of available HealthCheck check lists\n\n",
    )

    group1.add_argument(
        "--checklist_by_domain",
        action="store_true",
        default=None,
        help="Command to see list of available HealthCheck lists grouped by domain\n\n",
    )
    if not IS_POWERSCALE:
        group1.add_argument(
            "check_parameters",
            nargs="*",
            help="Command to provide parameters to check/checklist.\n"
            + "e.g --run pre_upgrade target_version 9.4.0.0\n\n",
        )

    group1.add_argument(
        "--checks",
        type=str,
        default=None,
        help="Command to see list of available checks for a specified "
        + "check list.\n"
        + "e.g --checks basic\n"
        + "Please use option '--checklist' to see available check lists\n\n",
    )

    group2.add_argument(
        "--group_by_nodes",
        action="store_true",
        default=False,
        help="Group results by node\n"
        + "Applies to view/view_by_domain, run/run_by_domain commands\n\n",
    )

    group2.add_argument(
        "--show_passed",
        action="store_true",
        default=False,
        help="When specified results include 'passed' checks\n"
        + "Applies to view/view_by_domain, run/run_by_domain commands\n\n",
    )

    group2.add_argument(
        "--show_unsupported",
        action="store_true",
        default=False,
        help="When specified results include 'unsupported' checks\n"
        + "Applies to view/view_by_domain, run/run_by_domain commands\n\n",
    )

    group2.add_argument(
        "--send_email",
        type=str,
        default=None,
        help="Specifiy comma separated list of destination email addresses.\n"
        + "e.g --send_email abc@dell.com,xyz@emc.com \n"
        + "Applies to view/view_by_domain, run/run_by_domain commands\n\n",
    )

    group2.add_argument(
        "--filepath",
        type=str,
        default=None,
        help="Output filepath to save results as HTML.\n"
        + "e.g --filepath /ifs/data/IsilonSupport/ \n"
        + "Applies to view/view_by_domain, run/run_by_domain commands\n\n",
    )

    if IS_POWERSCALE:
        add_historical_view_options(parser)

    parser.add_argument(
        "--verbose",
        action="store_true",
        default=False,
        help="When specified additional detailed debug messages get "
        + "displayed in the console",
    )
    if not IS_POWERSCALE:
        add_offcluster_options(parser)

    args = parser.parse_args()
    if (
        not IS_POWERSCALE
        and not args.logset
        and not (args.checks or args.checklist_by_domain or args.checklist)
    ):
        err_msg = "Logset is mandatory for off-cluster evaluation. --logset not specified.".format()
        logger.error(err_msg)
        os._exit(1)

    run(args)

    os._exit(0)


if __name__ == "__main__":
    main()
