#!/bin/bash
#
# File:   asrexachk
# MOS doc: https://support.oracle.com/epmos/faces/DocumentDisplay?id=1450112.1
#
VERSION="ASRExachk 3.4"

#########################################
if [ $(/usr/bin/id -u) -ne 0 ]; then
    echo Must be executed as the root user
    exit 1
fi
#########################################
#File Locations
#Exadata/Oracle Super Cluster
    #/opt/oracle.SupportTools/
#Exalogic
    #/opt/exalogic.tools/
#Exalytics
    #/opt/exalytics/
#BDA
    #/opt/oracle.SupportTools
#VARS
#########################################
#Debug Variable
DEBUG=false
#Log File Location
LOG_DIR=/var/log/asrexachk
LOG=$LOG_DIR/asrexachk.log
#Log Roll Variable
#clear log file if greater than 50K
if [ -e $LOG ]; then
    echo ""
else
    #check to see if directory exists
    if [ ! -d  $LOG_DIR ]; then
        mkdir $LOG_DIR
    fi
    echo "`date` File Start" > $LOG
fi
file_size_kb=`du -k "$LOG" | cut -f1`
if [ $file_size_kb -gt "500000" ]; then
    rm -f $LOG
    echo "Log File Rolled 'date;" > $LOG
fi
#########################################
# Global Vars
#########################################
SCRIPT=$(basename $0)
ZIP_FILES=false # set to true for quick downloading of test results
HOSTNAME=`hostname | /usr/bin/awk -F. '{ print $1 }'`
PMDIR=$HOSTNAME
LOGDIR=$LOG_DIR/$PMDIR

# 21105875
ASR_TEST_VALUE="False"
ASR_TEST_VALUE_OS="False"
ASR_TEST_VALUE_ILOM="False"

# 21182773
ASR_PORT=162

#########################################
#OS Validation Steps to determine OS Type for Processing of Exe locations differences
#OS Specific VARS
#Check for "Solaris 11" or "Solaris 11 Express" and then if "Oracle Enterprise Linux"
uname -a | grep -i "SunOS" > /dev/null 2>&1
if [ $? -eq 0 ] ; then
    #Check to see if Solaris 11 and exit script with error
    uname -r | grep "5.11" > /dev/null 2>&1
    if [ $? -eq 0 ]; then
        #"Solaris 11" or "Solaris 11 Express" executable location
        IPMILOC="/opt/ipmitool/bin"
        OSTYPE="SOLARIS"
        GGREP=ggrep
        SNMPTRAPCMD="/usr/bin/snmptrap"
    else
        echo "This is not a supported Solaris Version, Please contact Oracle Support"
    fi
else
    #"Oracle Enterprise Linux" executable locations
    uname -a | grep -i "Linux" > /dev/null 2>&1
    if [ $? -eq 0 ] ; then
        IPMILOC="/usr/bin"
        # 21569208
        if [ ! -f $IPMILOC/ipmitool ]; then
          IPMILOC="/usr/sbin"
        fi
        OSTYPE="LINUX"
        GGREP=grep
        SNMPTRAPCMD="/usr/bin/snmptrap"
    else
        #Error out if on a non-Solaris or Non-Linux Box 
        echo "This is not a supported OS Version or is a corrupted version of OS, Please contact Oracle Support"
        exit 1
    fi
fi

#Return Serial Number
if [ $OSTYPE = "LINUX" ]; then
    SERIAL_NUM=`dmidecode -s system-serial-number | sed 's/ *$//g' | grep -v "#"` #Bug 20371200
    SERVER_TYPE=`dmidecode -s system-product-name | sed 's/ *$//g' | grep -v "#" | tr ' ' '-'` #Bug 20371200
    SNMP_SERVER=`dmidecode -s system-product-name | sed 's/ *$//g' | grep -v "#"`
fi
if [ $OSTYPE = "SOLARIS" ]; then
    SERIAL_NUM=`$IPMILOC/ipmitool sunoem cli "show /SYS product_serial_number" | grep "product_serial_number" | /usr/bin/awk -F= '{ print $2 }'| sed -e 's/^[ \t]*//' | tr -d '\n'`
    SERVER_TYPE=`prtdiag | grep "System Configuration:" | awk -F: '{ print $2 }' | sed -e 's/^[ \t]*//' | sed 's/ *$//g' | tr ' ' '-'`
    SNMP_SERVER=`prtdiag | grep "System Configuration:" | awk -F: '{ print $2 }' | sed -e 's/^[ \t]*//' | sed 's/ *$//g'`
fi
#Output Variables
#Bug 20352697
# output files per section
OUT_SCRIPT="${LOGDIR}/asrexachk_$SERIAL_NUM.out"
OUT_GENERIC="${LOGDIR}/generic.out"
OUT_ILOM="${LOGDIR}/ilom.out"
OUT_DISKS="${LOGDIR}/disks.out"
OUT_FLASH="${LOGDIR}/flash.out"
OUT_STORAGE="${LOGDIR}/storage.out" # b items
OUT_DEBUG="${LOGDIR}/debug"
#End Bug 20352697
# End Global Vars
    # clean up from last run
    if $ZIP_FILES && [ -f $ZIPOUT ]; then
      rm $ZIPOUT
    fi
    if [ -d $LOGDIR ]; then
      rm -r $LOGDIR
    fi

    mkdir $LOGDIR
#########################################
#Global Case
# cli options: $1 is -f, -z, -p, -h / $2 is optional = ip:port for exalogic, if called from pm_wrapper
#
if [ -z $1 ]; then
    #used to add the function of checking configuration with test events
    TESTFUNCTION="TRUE"
    PMFUNCTION="FALSE"
    PM_CHECK="Standard ASRExachk"
elif [ -n $1 ]; then
    case $1 in
        -f|-F)
            TESTFUNCTION="FALSE"
            PMFUNCTION="FALSE" #20138670
            echo `date` "Running asrexachk via -f Option" >> $LOG
        ;;
        -z|-Z)
            #Running of the PM Check With ASR
            PMFUNCTION="TRUE" #20138670
            TESTFUNCTION="TRUE"
            echo `date` "Running asrexachk Proactive" >> $LOG
            PM_CHECK="Proactive ASRExachk" 
        ;;
        -p|-P)
            #Running of the PM Check Without ASR
            PMFUNCTION="TRUE"
            TESTFUNCTION="FALSE"
            echo `date` "Running asrexachk Proactive without ASR Testraps" >> $LOG
        ;;
        -s|-S)
            #To specify a non-default SNMP port (default = 162)
            if [ "$#" -gt 1 ]; then
              if [[ $2 =~ ^[0-9]{1,3}$ ]]; then
                  ASR_PORT=$2
                  echo `date` "Running asrexachk with SNMP port variable set to $ASR_PORT" >> $LOG
              else
                  echo `date` "$2 is not a valid SNMP port, using default port $ASR_PORT" >> $LOG
              fi
            fi
        ;;
        #help section
        -h|-H|*)
            echo "$VERSION"
            echo ""
            echo "NAME"
            echo -e '\t' "asrexachk - Automated Service Request (ASR) Engineered System"
            echo -e '\t' "validation and Proactive Maintenance script."
            echo ""
            echo "DESCRIPTION"
            echo ""
            echo "Running the asrexachk command validates the ASR configuration and performs"
            echo "a test from the Operating System and ILOM of the server that will result"
            echo "in an email coming from both telemetry sources, to the customer."
            echo "If either the emails are received an/or configuration errors are reported."
            echo ""
            echo "In addition there are option to perform a Proactive Maintenance check on the hardware"
            echo "and determine if there are any pre-existing issues and report out on them."
            echo ""
            echo "NOTE: The Proactive Maintenance reported results will need to have amanual"
            echo "Service Request generated as ASR does not report out on pre-exisiting issues."
            echo ""
            echo ""
            echo "OPTIONS"
            echo ""
            echo "In addition to running the "asrexachk" script with out any options there are some additional"
            echo "options that areavailable to add or suppress some functionality."
            echo ""
            echo -e '\t' "-h[H] Display a usage message on available options."
            echo ""
            echo -e '\t' "-f[F] Provides ASR validation only with NO ASR test events"
            echo -e '\t' "      and NO Proactive Maintenance"
            echo ""
            echo -e '\t' "-p[P] ASR & Proactive Maintenance but no ASR test events"
            echo ""
            echo -e '\t' "-s[S] Specify a SNMP port (default 162)"
            echo ""
            echo -e '\t' "-z[Z] ASR with test events and Proactive Maintenance"
            echo ""
            exit 0
        ;;
    esac
fi


# fail / pass coloring
PASS='\033[40;1;32m'
WARNING='\033[40;1;33m'
NO_COLOR='\e[0m'
#######################################
#SNMPTrap OID input
#OID Values are preset as Exadata does not use current MIB with OID Value for .80
# OID's
sunHwTrapFaultMessageID=".1.3.6.1.4.1.42.2.175.103.2.1.17.0"
sunHwTrapFaultDiagnosed=".1.3.6.1.4.1.42.2.175.103.2.0.80"
sunHwTrapSuspectComponentName=".1.3.6.1.4.1.42.2.175.103.2.1.6"
sunHwTrapProductManufacturer=".1.3.6.1.4.1.42.2.175.103.2.1.30"
sunHwTrapProductName=".1.3.6.1.4.1.42.2.175.103.2.1.15"
sunHwTrapProductSn=".1.3.6.1.4.1.42.2.175.103.2.1.31"
sunHwTrapSystemIdentifier=".1.3.6.1.4.1.42.2.175.103.2.1.1"
sunHwTrapHostname=".1.3.6.1.4.1.42.2.175.103.2.1.29"
sunHwTrapProductPn=".1.3.6.1.4.1.42.2.175.103.2.1.32"
sunHwTrapChassisManufacturer=".1.3.6.1.4.1.42.2.175.103.2.1.33"
sunHwTrapChassisName=".1.3.6.1.4.1.42.2.175.103.2.1.34"
sunHwTrapChassisSn=".1.3.6.1.4.1.42.2.175.103.2.1.35"
sunHwTrapChassisPn=".1.3.6.1.4.1.42.2.175.103.2.1.36"
sunHwTrapTopProductManufacturer=".1.3.6.1.4.1.42.2.175.103.2.1.37"
sunHwTrapTopProductName=".1.3.6.1.4.1.42.2.175.103.2.1.38"
sunHwTrapTopProductSn=".1.3.6.1.4.1.42.2.175.103.2.1.39"
sunHwTrapTopProductPn=".1.3.6.1.4.1.42.2.175.103.2.1.40"
sunHwTrapFirmwareVersion=".1.3.6.1.4.1.42.2.175.103.2.1.44"
sunHwTrapFirmwareBuild=".1.3.6.1.4.1.42.2.175.103.2.1.45"
sunHwTrapFirmwareDate=".1.3.6.1.4.1.42.2.175.103.2.1.46"
sunHwTrapILOMVersion=".1.3.6.1.4.1.42.2.175.103.2.1.49"
sunHwTrapAdditionalInfo=".1.3.6.1.4.1.42.2.175.103.2.1.9"
snmpTrapAddress=".1.3.6.1.6.3.18.1.3.0"
#End SNMPTrap OID Values
#Pull up time for Trap
UPTIME=`uptime | awk '{ print $3 }'`
#ASR_SNMP=""
#ASR_SNMP_DESC=""
#######################################
# Purpose: prints a output to screen and OUT_SCRIPT file
# Globals: OUT_SCRIPT
# Arguments: $1 = the output to print
# Returns: None
#######################################
function feedback_to_screen {
  echo -e "$1" | tee -a $OUT_SCRIPT
}

function screen_output {
    echo -e '\t' "$1" | tee -a $OUT_SCRIPT
}
# Debug Method
    # if in debug mode save all command output and timestamps to debug/ subdirectory
    if $DEBUG; then
      feedback_to_screen "== Running in debug mode =="
      feedback_to_screen "To profile, run: \n$ paste $OUT_DEBUG/$SCRIPT.$$.{tim,log}\n"
      mkdir $OUT_DEBUG
      if [ "$OSTYPE" == "LINUX" ]; then
        exec 3>&2 2> >(tee $OUT_DEBUG/$SCRIPT.$$.log | sed -u 's/^.*$/now/' | date -f - +%s.%N >$OUT_DEBUG/$SCRIPT.$$.tim) 
      else 
        # solaris sed -u and date -f not supported
        exec 3>&2 2> >(tee $OUT_DEBUG/$SCRIPT.$$.log | sed 's/^.*$/now/' | date >$OUT_DEBUG/$SCRIPT.$$.tim) 
      fi
    set -x
    fi
# End Debug Method


# Bug 21547362 - EXIT SCRIPT WHEN ASR IS NOT CONFIGURED 
function asr_not_configured {
  feedback_to_screen ""
  screen_output "${WARNING}#########Warning#########${NO_COLOR}"
  screen_output "ASR is not configured for this system"
  exit
}


#########################################
#Assign Device Type for Device methods
#########################################
#Pull Device Type
#Check BMC First
$IPMILOC/ipmitool bmc info > /dev/null 2>&1
if [ $? -eq 0 ]; then
    #Pull Device type
    IPMI_VERSION=`$IPMILOC/ipmitool -V | /usr/bin/awk '{ print $3 }'`
    if [ $IPMI_VERSION != "1.8.10.4" ]; then
        IPMI_BASE_1="`echo $IPMI_VERSION | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
        IPMI_BASE_2="`echo $IPMI_VERSION | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
        IPMI_BASE_3="`echo $IPMI_VERSION | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
        IPMI_BASE_4="`echo $IPMI_VERSION | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
        if [ $IPMI_BASE_1 -ge "1" ]; then
            if [ $IPMI_BASE_2 -ge "8" ]; then
                if [ $IPMI_BASE_3 -ge "10" ]; then
                    if [ $IPMI_BASE_3 -le "14" ]; then
                        if [ -z "$IPMI_BASE_4" ] || [ $IPMI_BASE_4 -ge "3" ]; then
                            DEV_TYPE=`$IPMILOC/ipmitool sunoem cli "show /SP system_identifier" | grep -i "system_identifier =" | /usr/bin/awk -F= '{ print $2 }' | sed -e 's/^[ \t]*//'`
                        else
                            #pull ILOM IP to pull system Identifer if possible
                            if [ $OSTYPE = "LINUX" ]; then
                                DEV_TYPE=`$IPMILOC/ipmitool sunoem cli "show /SP system_identifier" | grep -i "system_identifier =" | /usr/bin/awk -F= '{ print $2 }' | sed -e 's/^[ \t]*//'`
                            else
                                if [ -x /usr/sbin/asradm ]; then
                                    DEV_TYPE=`asradm send -n audit | grep "<product-id>" | /usr/bin/awk -F'>' '{ print $2 }' | /usr/bin/awk -F'<' '{ print $1 }'`
                                else
                                    DEV_TYPE=`$IPMILOC/ipmitool sunoem cli "show /SP system_identifier" | grep -i "system_identifier =" | /usr/bin/awk -F= '{ print $2 }' | sed -e 's/^[ \t]*//'`
                                fi
                            fi
                        fi
                    fi
                fi
            fi
        fi
    else
        DEV_TYPE=`$IPMILOC/ipmitool sunoem cli "show /SP system_identifier" | grep -i "system_identifier =" | /usr/bin/awk -F= '{ print $2 }' | sed -e 's/^[ \t]*//'`
    fi
else
    feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
    feedback_to_screen "Unable to pull device type through the BMC"
    feedback_to_screen "Please Contact Oracle Support for BMC Communication issues"
    feedback_to_screen "Once corrected, re-run ASREXACHK"
  
fi
#########################################
#Check to see if Exadata
echo "$DEV_TYPE" | egrep -i "Database Machine|Exadata Storage Expansion|ZDLRA|Zero Data Loss Recovery Appliance|Oracle Exadata" > /dev/null 2>&1
if [ $? -eq 0 ]; then
    METHOD_TYPE="Exadata"
    #########################################
    #General Exadata File and Executable locations
    EXADATA_MON="/opt/oracle.cellos/compmon"
    EXA_COMMAND="$EXADATA_MON/exadata_mon_hw_asr.pl"
    EXA_IMAGE=`/opt/oracle.cellos/imageinfo -ver`
    EXA_CELL_IMAGE=`/opt/oracle.cellos/imageinfo | grep "Active image version:" | awk -F: '{ print $2 }' | tr -d ' '`
    EXA_DB_IMAGE=`/opt/oracle.cellos/imageinfo | grep -i "Image version:"` #Bug 20352707
    EXA_ONECOMMAND="/opt/oracle.SupportTools/onecommand/onecommand.params"
    IMAGEINFO=`/opt/oracle.cellos/imageinfo`
    IMAGEINFO_NODE=`/opt/oracle.cellos/imageinfo -node 2>/dev/null`
    #MegaCLI Path Location
    if [ $OSTYPE = "SOLARIS" ]; then
        MEGACLI="/opt/MegaRAID/MegaCli"
    else
        if [ $OSTYPE = "LINUX" ]; then
            MEGACLI="/opt/MegaRAID/MegaCli/MegaCli64"
        fi
    fi
    # Update to use dbmcli Bug 20182652
    # Pull Image Info and separate it into single bases for validation checking
    IMAGEINFO_BASE=`echo "$EXA_DB_IMAGE" | awk -F: '{ print $2 }' | tr -d ' '`
    IMAGEINFO_BASE_1="`echo $EXA_IMAGE | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
    IMAGEINFO_BASE_2="`echo $EXA_IMAGE | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
    IMAGEINFO_BASE_3="`echo $EXA_IMAGE | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"

    #Validate Image version against 12.1.2.X to see if dbmcli is used
    #Prior versions to 12.1.2.X will use perl scripts for validation
    if [ $IMAGEINFO_BASE_1 -ge "12" ]; then
        if [ $IMAGEINFO_BASE_2 -ge "1" ]; then
            if [ $IMAGEINFO_BASE_3 -ge "2" ]; then
                DBMCLI="TRUE"
            else
                DBMCLI="FALSE"
            fi
        fi
    else
        DBMCLI="FALSE"
    fi
    # End update for dbmcli
    #########################################
else
    #check to see if BDA
    echo "$DEV_TYPE" | grep -i "Big Data"
    if [ $? -eq 0 ]; then
        METHOD_TYPE="BDA"
        #########################################
        #General BDA File and Executable locations
        EXADATA_MON="/opt/oracle/bda/compmon"
        EXA_COMMAND="$EXADATA_MON/bda_mon_hw_asr.pl"
        EXA_DB_IMAGE=`/opt/oracle/bda/bin/imageinfo | grep "IMAGE_VERSION             :"`
        # 20229882
        if [ -f /opt/oracle/bda/install/state/config.json ]; then
            INSTALL_INFO="/opt/oracle/bda/install/state/config.json"
        else
            INSTALL_INFO="/opt/oracle/bda/install/state/mammoth-saved.params"
        fi
        MEGACLI="/opt/oracle/bda/bin/MegaCli64"
        #########################################
    else
        #check to see if Exalytics
        echo "$DEV_TYPE" | grep -i "Exalytics"
        if [ $? -eq 0 ]; then
            METHOD_TYPE="Exalytics"
            #########################################
            #General Exalytics File and Executable locations
            EXADATA_MON="/opt/exalytics/asr"
            EXA_COMMAND="$EXADATA_MON/bda_mon_hw_asr.pl"
            EXA_DB_IMAGE=`/usr/sbin/imageinfo | grep "Image version       :"`
            MEGACLI="/opt/MegaRAID/MegaCli/MegaCli64"
            #########################################
        else
            #check to see if Exalogic
            echo "$DEV_TYPE" | grep -i "Exalogic"
            if [ $? -eq 0 ]; then
                METHOD_TYPE="Exalogic"
                #########################################
                #General Exalogic File and Executable locations
                EXA_CELL_IMAGE=`/usr/sbin/imageinfo | grep "Image version       :"`
                MEGACLI="/opt/MegaRAID/MegaCli/MegaCli64"
                # alternative megacli binary
                if [ ! -f $MEGACLI ]; then
                  MEGACLI="/opt/MegaRAID/storcli/storcli64"
                  if [ ! -f $MEGACLI ]; then
                    MEGACLI="/opt/MegaRAID/MegaCli"
                  fi
                fi
                EXADATA_MON="/opt/exalogic.tools/tools"
                if [ ! -d $EXADATA_MON ]; then
                  EXADATA_MON="/opt/exalogic/tools"
                fi

                #########################################
            else
                #check to see if SuperCluster
                echo "$DEV_TYPE" | grep -i "Supercluster"
                if [ $? -eq 0 ]; then
                    METHOD_TYPE="SSC"
                    #########################################
                    #General Supercluster File and Executable locations
                    #MegaCLI Path Location
                    if [ $OSTYPE = "SOLARIS" ]; then
                        MEGACLI="/opt/MegaRAID/MegaCli"
                    else
                        if [ $OSTYPE = "LINUX" ]; then
                            MEGACLI="/opt/MegaRAID/MegaCli/MegaCli64"
                        fi
                    fi
                    EXADATA_MON="/opt/oracle.SupportTools"
                    IMAGEINFO_PATH="/opt/oracle.cellos/imageinfo"
                    if [ ! -f $IMAGEINFO_PATH ]; then
                      IMAGEINFO_PATH="/usr/local/bin/imageinfo"
                    fi
                    IMAGEINFO=`$IMAGEINFO_PATH 2>/dev/null`
                    IMAGEINFO_NODE=`$IMAGEINFO_PATH -node 2>/dev/null`
                    #########################################
                else
                    echo -e '\t' "Unable to determine Device Type, System Identifier states:"
                    echo -e '\t' "--> $DEV_TYPE <--"
                    echo -e '\t' "If system is returning a null value or (None)"
                    echo -e '\t' "Please open a Service Request with the Exadata Hardware Team"
                    echo -e '\t' "To help set your system Identifier."
                    echo `date` "Unable to determine Device Type" >> $LOG
                    echo `$IPMILOC/ipmitool sunoem cli "show /SP system_identifier" | grep -i "system_identifier ="` >> $LOG
                    exit 1
                fi
            fi
        fi
    fi
fi
#END Assign Device Type for Device methods
#########################################
#Status return variable Function Checks
#Standard Error Outputs
ASR_Return_1() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}" 
    echo -e '\t' "It is recommended that the ASR Manager IP should be"
    echo -e '\t' "on a management type network that does not carry client access traffic."
    echo -e '\t' "Please see the ASR Security Guide for Ports and Protcol for Required Firewall"
    echo -e '\t' "Requirements for SNMP Traps to be sent to the ASR Manager"
    if [ $METHOD_TYPE == "Exadata" ]; then
        echo -e '\t' "Client Access/Bond bondeth*or Customer Private eth2 and or eth3 Networks."
        echo -e '\t' "The ASR Manager network traffic is recommended to exit through the Management interfaces,"
        echo -e '\t' "which is interface: Net0/eth0 and the ILOM port NetMGMT."
    elif [[ $METHOD_TYPE == "BDA" || "Exalytics" ]]; then
        echo -e '\t' "Client Access/Bond bondeth*or Customer Private eth2 and or eth3 Networks."
        echo -e '\t' "The ASR Manager network traffic is recommended to exit through the, which is interface:"
        echo -e '\t' "Net0/eth0 and the ILOM port NetMGMT."
    fi
    ASR_SNMP="ASR-8001-JA"
    ASR_SNMP_DESC="ASR_Return_1"
}
ASR_Return_2() {
    echo ""
    echo "Management IP Interface check completed"
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "ASR is not recommended on this node due to the Management Subnet "
    echo -e '\t' "overlapping multiple Interfaces on the Same Subnet."
    if [ $METHOD_TYPE == "Exadata" ]; then
        echo -e '\t' "Please check the Exadata Database Machine Owner's Guide"
        echo -e '\t' "or work with the Oracle Exadata Support Team"
        echo -e '\t' "on moving the Managment Interface off of the Client Access Subnet,"
        echo -e '\t' "Customer Private Network and/or Infiniband Private Network per "
        echo -e '\t' "Chapter 3 section 1 of the Exadata Database Machine Owner's Guide"
        echo -e '\t' "under the Overview of Network Requirements"
        echo -e '\t' "The ASR Manager is recommended to be on the Management interfaces, which is interface:"
        echo -e '\t' "Net0/eth0 and the ILOM port NetMGMT."
    elif [[ $METHOD_TYPE == "BDA" || "Exalytics" ]]; then
        echo -e '\t' "Client Access/Bond bondeth*or Customer Private eth2 and or eth3 Networks."
        echo -e '\t' "The ASR Manager is recommended to be on the Management interfaces, which is interface:"
        echo -e '\t' "Net0/eth0 and the ILOM port NetMGMT."
    fi
    ASR_SNMP="ASR-8001-SO"
    ASR_SNMP_DESC="ASR_Return_2"
   
}
ASR_Return_3() {
    echo ""
    echo "Management IP Interface check completed"
    echo -e '\t' "ASR is supported on this Management Interface configuration"
}
ASR_Return_4() {
    echo ""
    echo "Management IP Interface check completed"
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "This is a non-standard configuration which ASR is not recommended on $INTFAIL"
    echo -e '\t' "Please follow instructions provided in the error above by Node IP interfce Check" 
    echo -e '\t' "for the Non-Standard Configuration"
    ASR_SNMP="ASR-8001-NW"
    ASR_SNMP_DESC="ASR_Return_4"
}
ASR_Return_5() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "Unable to determine configuration status of the DB node."
    echo -e '\t' "Please check to make sure the ASR Manager SNMP Port is"
    echo -e '\t' "setup through the command to include the port of 162"
    echo -e '\t' "Please reference the Oracle Exadata Administration guide for more"
    echo -e '\t' "details on DB commands for ASR"
    ASR_SNMP="ASR-8001-HI"
    ASR_SNMP_DESC="ASR_Return_5"
}
ASR_Return_6() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "IPADM is reporting interface $INTFAIL as a non-standard configuration."
    echo -e '\t' "Interface/? indicates that the address was configured"
    echo -e '\t' "on the interface outside of ipadm command set."
    echo -e '\t' "This is a non-standard configuration which ASREXACHK is unable to validate against"
    echo -e '\t' "Please work with the Oracle Exadata Support Team"
    echo -e '\t' "to resolve the network setup so the value is set to a standard configuration"
    ASR_SNMP="ASR-8001-TE"
    ASR_SNMP_DESC="ASR_Return_6"
}
ASR_Return_7() {
    echo -e "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "Unable to Validate SNMP Due to Unconfigured ASR Setup"
    echo -e '\t' "This is based on a IP Validation provided in previous Checks"
    echo -e '\t' "Please contact Oracle Exadata Support for help in resolving"
    echo -e '\t' "network configuration issue"
}
ASR_Return_8() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "Unable to Validate SNMP Due to Unconfigured ASR Setup"
    echo -e '\t' "Please check the errors provided in the Configuration Check"
    echo -e '\t' "Please contact Oracle Support for help in resolving configuration issue"
    ASR_SNMP="ASR-8001-11"
    ASR_SNMP_DESC="ASR_Return_8"
}
ASR_Return_9() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "The Fromip does not match what is expected on the interface of eth0."
    echo -e '\t' "Please run the SNMP setup with the from IP set to the management"
    echo -e '\t' "interface ip for this server node"
    ASR_SNMP="ASR-8001-20"
    ASR_SNMP_DESC="ASR_Return_9"
}
ASR_Return_10() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "The Fromip is set to a hostname value instead of an IP"
    echo -e '\t' "Please run the SNMP setup with the the fromip set to the management"
    echo -e '\t' "interface ip for this server node"
    ASR_SNMP="ASR-8001-09"
    ASR_SNMP_DESC="ASR_Return_10"
}
ASR_Return_11() {
    echo -e '\t' "${WARNING}#########Warning#########${NO_COLOR}"
    echo -e '\t' "The Fromip is not being used. If the Image is greater than 11.2.2.4.0"
    echo -e '\t' "Please run the SNMP setup with the from IP set to the management"
    echo -e '\t' "interface IP for this server node, else if it is on a previous version"
    echo -e '\t' "of the Exadata Image you can bypass this error message"
}
#End Status return Function checks
#########################################
#End VARS
#########################################
#Main Functions for status checks
#########################################
#Functions to pull ASR Manager IP's
#########################################
#Start Exadata ASR Manager Ips
#########################################
#ASR Manager IP pull for DB node
ASR_MAN_IP_DB() {
    #Global Var set
    DUAL_ASR_ENTRY=""
    # Bug 20182652
    if [ "$DBMCLI" == "TRUE" ]; then
        #Count instances of Type=asr. Type ASR 1 is allowed
        DB_TYPECOUNT=`dbmcli -e "list dbserver attributes snmpsubscriber" | grep -o -i "type=asr" | wc -l`
        DB_ASR_COUNT=`echo $DB_TYPECOUNT`
        if [ $DB_ASR_COUNT -eq "1" ]; then
            ASR_MAN=`dbmcli -e list dbserver detail | grep -i "type=asr" | grep "host=" | /usr/bin/awk -F, '{ print $1 }' | /usr/bin/awk -F= '{ print $2 }'`
            if [[ $ASR_MAN =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
                ASR_IP="$ASR_MAN"
            else
                ASR_IP=`/usr/bin/python -c 'import socket; print socket.gethostbyname("'$ASR_MAN'")'`
            fi
        else
                if [ $CELL_ASR_COUNT -gt "1" ]; then
                asr_not_configured 
                ASR_SETUP_STATUS="Unconfigured"
                ASR_IP="N/A"
                DUAL_ASR_ENTRY="TRUE"
            fi
        fi
    else
        #Check for file and if exists check count of asr trap destinations, proceed if no error
        if [ -f $EXADATA_MON/asrs.state ]; then
            grep -i $ASR_PORT $EXADATA_MON/asrs.state > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                ASR_IP_COUNT=`grep -i ":$ASR_PORT" $EXADATA_MON/asrs.state | wc -l`
                if [ $ASR_IP_COUNT -eq "1" ]; then
                    ASR_MAN=`cat $EXADATA_MON/asrs.state | grep -i ":$ASR_PORT" | /usr/bin/awk -F: '{ print $1 }'`
                    if [[ $ASR_MAN =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
                        ASR_IP="$ASR_MAN"
                    else
                        ASR_IP=`/usr/bin/python -c 'import socket; print socket.gethostbyname("'$ASR_MAN'")'`
                    fi
                    #ASR IP Expanded for checking
                    ASR_MAN_octet_1="`echo $ASR_IP | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    ASR_MAN_octet_2="`echo $ASR_IP | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    ASR_MAN_octet_3="`echo $ASR_IP | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    ASR_MAN_octet_4="`echo $ASR_IP | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                else
                    asr_not_configured 
                    ASR_SETUP_STATUS="Unconfigured"
                    ASR_IP="N/A"
                    DUAL_ASR_ENTRY="TRUE"
                fi
            else
                asr_not_configured 
                ASR_SETUP_STATUS="Unconfigured"
                ASR_IP="N/A"        
            fi
        fi
    fi
}
#ASR Manager IP pull for Cell node
ASR_MAN_IP_CELL() {
    #Count instances of Type=asr. Type ASR 1 is allowed
    CELL_TYPECOUNT=`cellcli -e "list cell attributes snmpsubscriber" | grep -o -i "type=asr" | wc -l`
    CELL_ASR_COUNT=`echo $CELL_TYPECOUNT`
    if [ $CELL_ASR_COUNT -eq "1" ]; then
        ASR_MAN=`cellcli -e list cell detail | grep -i "type=asr" | grep "host=" | /usr/bin/awk -F, '{ print $1 }' | /usr/bin/awk -F= '{ print $2 }'`
        if [[ $ASR_MAN =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
            ASR_IP="$ASR_MAN"
        else
            ASR_IP=`/usr/bin/python -c 'import socket; print socket.gethostbyname("'$ASR_MAN'")'`
        fi
    else
            if [ $CELL_ASR_COUNT -gt "1" ]; then
            asr_not_configured 
            ASR_SETUP_STATUS="Unconfigured"
            ASR_IP="N/A"
            DUAL_ASR_ENTRY="TRUE"
        fi
    fi
}
#End Exadata ASR Manager Ips
#########################################

#########################################
#END Functions to pull ASR Manager IP's
#########################################
# Netmask calculation Function (Solaris Only)
# Convert "/" notation to dotted decimal
netmask_calc(){
    if [ "$netmask" = "30" ]; then
        subnetmask="255.255.255.252"
    else
        if [ "$netmask" = "29" ]; then
            subnetmask="255.255.255.248"
        else
            if [ "$netmask" = "28" ]; then
                subnetmask="255.255.255.240"
            else
                if [ "$netmask" = "27" ]; then
                    subnetmask="255.255.255.224"
                else
                    if [ "$netmask" = "26" ]; then
                        subnetmask="255.255.255.192"
                    else
                        if [ "$netmask" = "25" ]; then
                            subnetmask="255.255.255.128"
                        else
                            if [ "$netmask" = "24" ]; then
                                subnetmask="255.255.255.0"
                            else
                                if [ "$netmask" = "23" ]; then
                                    subnetmask="255.255.254.0"
                                else
                                    if [ "$netmask" = "22" ]; then
                                        subnetmask="255.255.252.0"
                                    else
                                        if [ "$netmask" = "21" ]; then
                                            subnetmask="255.255.248.0"
                                        fi
                                    fi
                                fi
                            fi
                        fi
                    fi
                fi
            fi
        fi
    fi
}
# END Netmask calculation Function (Solaris Only)
#########################################
#Validate IP Configurations for Linux and Solaris and comapare to 
#Configurations as well as to the ASR Manager IP
#########################################
#Linux Configuration check
linux_ip_int_check() {
    #Reset of Generic Variables
    ASR_NET="0"
    netmask="0"
    subnetmask="0"
    
    
    #Pull ASR Manager IP if the state is configured
    if [ $ASR_SETUP_STATUS = "Configured" ]; then
        ASR_MAN_IP_DB
    fi
    TMPFILE=`mktemp -q -t ipcheck.XXXXXX`
    if [ ! -z "$TMPFILE" ]; then
        #Loop To check interfaces
        #Check of eth0 is not on same subnet as the bnd network
        for file in /etc/sysconfig/network-scripts/ifcfg-*
            do
                if [ $METHOD_TYPE != "Exalytics" ]; then
                    INTNAME=`echo $file | /usr/bin/awk -F/ '{print $5 }' | /usr/bin/awk -F- '{ print $2 }'` > /dev/null 2>&1
                    NETIP=`grep -i "NETWORK" $file | /usr/bin/awk -F= '{print $2 }'` > /dev/null 2>&1
                    BCASTIP=`grep -i "BROADCAST" $file | /usr/bin/awk -F= '{print $2 }'` > /dev/null 2>&1
                    echo $INTNAME "=" $NETIP "=" $BCASTIP >> $TMPFILE
                else
                    INTNAME=`echo $file | /usr/bin/awk -F/ '{print $5 }' | /usr/bin/awk -F- '{ print $2 }'` > /dev/null 2>&1
                    NETIP=`grep -i "IPADDR" $file | /usr/bin/awk -F= '{print $2 }'` > /dev/null 2>&1
                    BCAST=`ifconfig $INTNAME | grep -i Bcast | /usr/bin/awk '{ print $3 }'` > /dev/null 2>&1
                    BCASTIP=`echo $BCAST | /usr/bin/awk -F: '{ print $2 }'` > /dev/null 2>&1
                    echo $INTNAME"="$NETIP"="$BCASTIP >> $TMPFILE
                fi
            #Validate ASR Manager IP to the Interface in the loop
            if [ $ASR_SETUP_STATUS = "Configured" ]; then
                # Pull subnet base and Subnet Broad Cast to validate ASR IP to interface IP
                #ASR Manager Only allowed on eth0
                #Subnet IP Expanded for checking
                SUBNET_BASE_octet_1="`echo $NETIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                SUBNET_BASE_octet_2="`echo $NETIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                SUBNET_BASE_octet_3="`echo $NETIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                SUBNET_BASE_octet_4="`echo $NETIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"

                #Subnet BCast Expanded for checking
                SUBNET_BROADCAST_octet_1="`echo $BCASTIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                SUBNET_BROADCAST_octet_2="`echo $BCASTIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                SUBNET_BROADCAST_octet_3="`echo $BCASTIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                SUBNET_BROADCAST_octet_4="`echo $BCASTIP | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"

                #Checking ASR Manager IP Interface Against the rest of the Interfaces
                #ASR Manager is required not to be on the Same Subnet as the Client access network
                nonmgmt_int="$INTNAME"
                if [[ "$nonmgmt_int" != "eth0" ]] || [[ "$nonmgmt_int" != "vmeth0" ]]; then          
                    # checking to see if value is setup prior to validating interface
                    if [ ! -z "$NETIP" ]; then
                        screen_output "Validating ASR Manager IP to Interface $INTNAME"
                        if [[ $ASR_MAN_octet_1 -eq $SUBNET_BASE_octet_1 && $SUBNET_BROADCAST_octet_1 ]]; then
                            if [[ $ASR_MAN_octet_2 -eq $SUBNET_BASE_octet_2 && $SUBNET_BROADCAST_octet_2 ]]; then
                                if [ $ASR_MAN_octet_3 -ge $SUBNET_BASE_octet_3 ] && [ $ASR_MAN_octet_3 -le $SUBNET_BROADCAST_octet_3 ]; then
                                    if [ $ASR_MAN_octet_4 -ge $SUBNET_BASE_octet_4 ] && [ $ASR_MAN_octet_4 -le $SUBNET_BROADCAST_octet_4 ]; then
                                        ASR_NET="1"
                                        result=$(ASR_Return_1)
                                        feedback_to_screen "$result"
                                        feedback_to_screen ""
                                        ASR_SNMP="ASR-8001-JA"
                                        ASR_SNMP_DESC="ASR_Return_1"
                                        echo "interface failure $INTNAME" >> $TMPFILE
                                    fi
                                fi
                            fi
                        fi
                    fi
                else
                    ASR_NET="0"
                fi
            fi
        done
        #Validating the Subnet Count to eth0 and other activated intefaces
        #Pull eth0 Subnet IP into Variable
        ETH0=`grep -i "eth0" $TMPFILE | grep -v b | /usr/bin/awk -F= '{ print $2 }'`
        #Count Subnets that equal eth0 Subnet
        grep -i "vmeth0" $TMPFILE > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            NETCOUNT="1"
        else
            NETCOUNT=`grep -o "$ETH0" $TMPFILE | wc -l`
        fi
        #If subnet count is greater than one return value return code
        if [ $NETCOUNT -ne "1" ]; then
            ASR_NET="2"
        fi
        #Check Return Codes and report outputs
        if [ $ASR_NET -eq "1" ]; then
            INTFAIL=`grep "interface failure" $TMPFILE | /usr/bin/awk '{ print $3 }'`
            result=$(ASR_Return_4)
            feedback_to_screen "$result"
            ASR_SNMP="ASR-8001-NW"
            ASR_SNMP_DESC="ASR_Return_4"
        else
            if [ $ASR_NET -eq "2" ]; then
                result=$(ASR_Return_2)
                feedback_to_screen "$result"
                ASR_SNMP="ASR-8001-SO"
                ASR_SNMP_DESC="ASR_Return_2"
            else
                if [  $ASR_NET -eq "0" ]; then
                    result=$(ASR_Return_3)
                    feedback_to_screen "$result"
                fi
            fi
        fi
        #remove temp file
        rm -f $TMPFILE
    fi
}
#END Linux Configuration check
#########################################
#Solaris Configuration Check
solaris_ip_int_check() {
    #Reset of Generic Variables
    ASR_NET="0"
    netmask="0"
    subnetmask="0"
    
    TMPFILE=`mktemp -q -t ipcheck.XXXXXX`
    if [ ! -z "$TMPFILE" ]; then
        #Pull ASR Manager IP if the state is configured
        if [ $ASR_SETUP_STATUS = "Configured" ]; then
            ASR_MAN_IP_DB
        fi
        #Pull Solaris eth0/net0 Interface name
        ipadm show-if | grep -i "net" > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            NETINT=`ipadm show-addr | grep -i "net" | grep "v4" | /usr/bin/awk '{print $1}'`
        else
            ipadm show-if | grep -i "eth" > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                NETINT=`ipadm show-addr | grep -i "eth" | grep -v "bond" | grep -v "::" | /usr/bin/awk '{print $1}'`
            else
                result=$(ASR_Return_5)
                feedback_to_screen "$result"
                ASR_SNMP="ASR-8001-HI"
                ASR_SNMP_DESC="ASR_Return_5"
            fi
        fi
        #Loop To check interfaces
        #Check of eth0 is not on same subnet as the Customer private network
        feedback_to_screen ""
        feedback_to_screen "Validating the Private Network Interfaces"
        for i in $NETINT
            do
            #check interface for status not configured by ipadm
            ipadm show-addr $i | grep -i "/?" > /dev/null 2>&1
            if [ $? -ne 0 ]; then
                int=`ipadm show-addr $i -o addr | grep -v "ADDR"`
                IPADDR=`echo $int | /usr/bin/awk -F/ '{print $1}'`
                netmask=`echo $int | /usr/bin/awk -F/ '{print $2}'`
                if [ -n "$IPADDR" ]; then
                    ## Convert "/" notation to dotted decimal#
                    netmask_calc
                    ## Get the last octet of the subnet mask#
                    netmask_octet_4="`echo $subnetmask | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                    netmask_octet_3="`echo $subnetmask | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    ## Get first three octets of the IP address#
                    IPADDR_octet_1="`echo $IPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    IPADDR_octet_2="`echo $IPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    IPADDR_octet_3="`echo $IPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    IPADDR_octet_4="`echo $IPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                ## Get the size of the subnet#
                if [[ "$netmask" == "30" || "29" || "28" || "27" || "26" || "25" || "24" ]]; then
                    subnet_size=$[256-$netmask_octet_4]
                    ## Get the last octet of the network address
                    if [ "$netmask_octet_4" = "0" ]; then
                        subnet_base="0"
                    else
                        subnet_base1=$[$IPADDR_octet_4/$subnet_size]
                        subnet_base=$[$subnet_base1*$subnet_size]
                    fi

                    SUBNET_BASE=$IPADDR_octet_1.$IPADDR_octet_2.$IPADDR_octet_3.$subnet_base
                    SUBNET_BROADCAST=$IPADDR_octet_1.$IPADDR_octet_2.$IPADDR_octet_3.$[$subnet_base-1+$subnet_size]

                else

                    if [[ "$netmask" == "23" || "22" || "21" ]]; then
                        subnet_size=$[256-$netmask_octet_3]


                        ## Get the last octet of the network address#
                        if [ "$netmask_octet_3" = "0" ]; then
                            subnet_base="0"
                        else
                            subnet_base1=$[$IPADDR_octet_3/$subnet_size]
                            subnet_base=$[$subnet_base1*$subnet_size]
                        fi
                        SUBNET_BASE=$IPADDR_octet_1.$IPADDR_octet_2.$subnet_base.$IPADDR_octet_4
                        SUBNET_BROADCAST=$IPADDR_octet_1.$IPADDR_octet_2.$[$subnet_base-1+$subnet_size].$IPADDR_octet_4
                    fi
                fi
                #Write the outputs to file
                echo $i $IPADDR $SUBNET_BASE >> $TMPFILE
                #Validate ASR Manager IP to the Interface in the loop
                if [ $ASR_SETUP_STATUS = "Configured" ]; then
                    # Pull subnet base and Subnet Broad Cast to validate ASR IP to interface IP
                    #ASR Manager Only allowed on eth0
                    #Subnet IP Expanded for checking
                    SUBNET_BASE_octet_1="`echo $SUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    SUBNET_BASE_octet_2="`echo $SUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    SUBNET_BASE_octet_3="`echo $SUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    SUBNET_BASE_octet_4="`echo $SUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                    #Subnet BCast Expanded for checking
                    SUBNET_BROADCAST_octet_1="`echo $SUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    SUBNET_BROADCAST_octet_2="`echo $SUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    SUBNET_BROADCAST_octet_3="`echo $SUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    SUBNET_BROADCAST_octet_4="`echo $SUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                    #Checking ASR Manager IP Interface Against the rest of the Interfaces
                    #ASR Manager is required not to be on the Same Subnet as the Client access network
                    nonmgmt_int="`echo $i | /usr/bin/awk -F/ '{ print $1}'`"
                    if [ "$nonmgmt_int" != "eth0" ]; then
                        # checking to see if value is setup prior to validating interface
                        screen_output "Validating ASR Manager IP to Interface $i"
                        if [[ $ASR_MAN_octet_1 -eq $SUBNET_BASE_octet_1 && $SUBNET_BROADCAST_octet_1 ]]; then
                            if [[ $ASR_MAN_octet_2 -eq $SUBNET_BASE_octet_2 && $SUBNET_BROADCAST_octet_2 ]]; then
                                if [ $ASR_MAN_octet_3 -ge $SUBNET_BASE_octet_3 ] && [ $ASR_MAN_octet_3 -le $SUBNET_BROADCAST_octet_3 ]; then
                                    if [ $ASR_MAN_octet_4 -ge $SUBNET_BASE_octet_4 ] && [ $ASR_MAN_octet_4 -le $SUBNET_BROADCAST_octet_4 ]; then
                                        ASR_NET="1"
                                        result=$(ASR_Return_1)
                                        feedback_to_screen "$result"
                                        feedback_to_screen ""
                                        ASR_SNMP="ASR-8001-JA"
                                        ASR_SNMP_DESC="ASR_Return_1"
                                        echo "interface failure $i" >> $TMPFILE
                                    fi
                                fi
                            fi
                        fi
                    else
                        ASR_NET="0"
                    fi
                fi
            fi
        else
            ASR_NET="3"
            result=$(ASR_Return_6)
            feedback_to_screen "$result"
            feedback_to_screen ""
            ASR_SNMP="ASR-8001-TE"
            ASR_SNMP_DESC="ASR_Return_6"
            echo "temp interface $i" >> $TMPFILE
        fi
        done
        #Loop To check interfaces
        #Check of eth0 is not on same subnet as the Bonded network
        feedback_to_screen ""
        feedback_to_screen "Validating the Bonded Network Interfaces"
        #Reset of Generic Variables
        netmask="0"
        subnetmask="0"
        #Pull Solaris Bonded Interface names
        BNDINT=`ipadm show-addr | grep -i "bond" | grep -v "::" | /usr/bin/awk '{print $1}'`
        #Loop To check interfaces
        #Check of eth0 is not on same subnet as the BOND network
        for j in $BNDINT
            do
            #check interface for status not configured by ipadm
            ipadm show-addr $j | grep -i "/?" > /dev/null 2>&1
            if [ $? -ne 0 ]; then
                    bndint=`ipadm show-addr $j -o addr | grep -v -i "ADDR"`
                    BNDIPADDR=`echo $bndint | /usr/bin/awk -F/ '{print $1}'`
                    netmask=`echo $bndint | /usr/bin/awk -F/ '{print $2}'`
                    BNDnetmask=$netmask
                if [ -n "$IPADDR" ]; then
                    ## Convert "/" notation to dotted decimal#
                    netmask_calc
                    ## Get the last octet of the subnet mask#
                    BNDnetmask_octet_4="`echo $subnetmask | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                    BNDnetmask_octet_3="`echo $subnetmask | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    ## Get first three octets of the IP address#
                    BNDIPADDR_octet_1="`echo $BNDIPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    BNDIPADDR_octet_2="`echo $BNDIPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    BNDIPADDR_octet_3="`echo $BNDIPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    BNDIPADDR_octet_4="`echo $BNDIPADDR | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                ## Get the size of the subnet#
                if [[ "$BNDnetmask" = "30" || "29" || "28" || "27" || "26" || "25" || "24" ]]; then
                    BNDsubnet_size=$[256-$BNDnetmask_octet_4]
                    ## Get the last octet of the network address
                    if [ "$BNDnetmask_octet_4" == "0" ]; then
                        BNDsubnet_base="0"
                    else
                        BNDsubnet_base1=$[$BNDIPADDR_octet_4/$BNDsubnet_size]
                        BNDsubnet_base=$[$BNDsubnet_base1*$BNDsubnet_size]
                    fi
                    BNDSUBNET_BASE=$BNDIPADDR_octet_1.$BNDIPADDR_octet_2.$BNDIPADDR_octet_3.$BNDsubnet_base
                    BNDSUBNET_BROADCAST=$BNDIPADDR_octet_1.$BNDIPADDR_octet_2.$BNDIPADDR_octet_3.$[$BNDsubnet_base-1+$BNDsubnet_size]
                else
                    if [[ "$BNDnetmask" = "23" || "22" || "21" ]]; then
                        BNDsubnet_size=$[256-$BNDnetmask_octet_3]
                        ## Get the last octet of the network address#
                        if [ "$BNDnetmask_octet_3" == "0" ]; then
                            BNDsubnet_base="0"
                        else
                            BNDsubnet_base1=$[$BNDIPADDR_octet_3/$BNDsubnet_size]
                            BNDsubnet_base=$[$BNDsubnet_base1*$BNDsubnet_size]
                        fi

                        BNDSUBNET_BASE=$BNDIPADDR_octet_1.$BNDIPADDR_octet_2.$BNDsubnet_base.$BNDIPADDR_octet_4
                        BNDSUBNET_BROADCAST=$BNDIPADDR_octet_1.$BNDIPADDR_octet_2.$BNDIPADDR_octet_3.$[$BNDsubnet_base-1+$BNDsubnet_size]
                    fi
                fi
                #Write the outputs to file
                echo $j $BNDIPADDR $BNDSUBNET_BASE >> $TMPFILE
                #Validate ASR Manager IP to the Interface in the loop
                if [ $ASR_SETUP_STATUS = "Configured" ]; then
                    # Pull subnet base and Subnet Broad Cast to validate ASR IP to interface IP
                    #ASR Manager Only allowed on eth0
                    #Subnet IP Exapanded for checking
                    BNDSUBNET_BASE_octet_1="`echo $BNDSUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    BNDSUBNET_BASE_octet_2="`echo $BNDSUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    BNDSUBNET_BASE_octet_3="`echo $BNDSUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    BNDSUBNET_BASE_octet_4="`echo $BNDSUBNET_BASE | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                    #Subnet BCast Expanded for checking
                    BNDSUBNET_BROADCAST_octet_1="`echo $BNDSUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $1}'`"
                    BNDSUBNET_BROADCAST_octet_2="`echo $BNDSUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $2}'`"
                    BNDSUBNET_BROADCAST_octet_3="`echo $BNDSUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $3}'`"
                    BNDSUBNET_BROADCAST_octet_4="`echo $BNDSUBNET_BROADCAST | sed -e 's/\./ /g' | /usr/bin/awk '{print $4}'`"
                    #Checking ASR Manager IP Interface
                    nonmgmt_int="`echo $j | /usr/bin/awk -F/ '{ print $1}'`"
                    if [ "$nonmgmt_int" != "eth0" ]; then
                        # checking to see if value is setup prior to validating interface
                        screen_output "Validating ASR Manager IP to Interface $j"
                        if [[ $ASR_MAN_octet_1 -eq $BNDSUBNET_BASE_octet_1 && $BNDSUBNET_BROADCAST_octet_1 ]]; then
                            if [[ $ASR_MAN_octet_2 -eq $BNDSUBNET_BASE_octet_2 && $BNDSUBNET_BROADCAST_octet_2 ]]; then
                                if [ $ASR_MAN_octet_3 -ge $BNDSUBNET_BASE_octet_3 ] && [ $ASR_MAN_octet_3 -le $BNDSUBNET_BROADCAST_octet_3 ]; then
                                    if [ $ASR_MAN_octet_4 -ge $BNDSUBNET_BASE_octet_4 ] && [ $ASR_MAN_octet_4 -le $BNDSUBNET_BROADCAST_octet_4 ]; then
                                        ASR_NET="1"
                                        result=$(ASR_Return_1)
                                        feedback_to_screen "$result"
                                        feedback_to_screen ""
                                        ASR_SNMP="ASR-8001-JA"
                                        ASR_SNMP_DESC="ASR_Return_1"
                                        echo "interface failure $j" >> $TMPFILE
                                    fi
                                fi
                            fi
                        fi
                    else
                        ASR_NET="0"
                    fi
                fi
            fi
        else
            ASR_NET="3"
            result=$(ASR_Return_6)
            feedback_to_screen "$result"
            feedback_to_screen ""
            ASR_SNMP="ASR-8001-TE"
            ASR_SNMP_DESC="ASR_Return_6"
            echo "temp interface $j" >> $TMPFILE
        fi
        done
        #Validating the Subnet Count to eth0 and other activated intefaces
        #Pull eth0 Subnet IP into Variable for net0 device name
        grep -i "net0" $TMPFILE > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            SOL11NETCOUNT=`grep -i "net0" $TMPFILE | grep -v bond | /usr/bin/awk '{print $3 }' | wc -l`
            if [ $SOL11NETCOUNT -ne "1" ]; then
                ASR_NET="2"
            fi
        else
            #Pull eth0 Subnet IP into Variable for net0 device name
            SOL11NETCOUNT=`grep -i "eth0/" $TMPFILE | grep -v bond |  /usr/bin/awk '{print $3 }' | wc -l`
            if [ $SOL11NETCOUNT -ne "1" ]; then
                ASR_NET="2"
            fi
        fi
        #Check Return Codes and report outputs
            if [ $ASR_NET -eq "1" ]; then
                INTFAIL=`grep "interface failure" $TMPFILE | /usr/bin/awk '{ print $3 }'`
                result=$(ASR_Return_4)
                feedback_to_screen "$result"
                ASR_SNMP="ASR-8001-NW"
                ASR_SNMP_DESC="ASR_Return_4"
            else
                if [ $ASR_NET -eq "2" ]; then
                    result=$(ASR_Return_2)
                    feedback_to_screen "$result"
                    ASR_SNMP="ASR-8001-SO"
                    ASR_SNMP_DESC="ASR_Return_2"
                else
                    if [  $ASR_NET -eq "0" ]; then
                        result=$(ASR_Return_3)
                        feedback_to_screen "$result"
                    else
                        if [ $ASR_NET -eq "3" ]; then
                            INTFAIL=`grep "temp interface" $TMPFILE | /usr/bin/awk '{ print $3 }'`
                            result=$(ASR_Return_4)
                            feedback_to_screen "$result"
                            ASR_SNMP="ASR-8001-NW"
                            ASR_SNMP_DESC="ASR_Return_4"
                        fi
                    fi
                fi
            fi
        #remove temp file
        rm -f $TMPFILE
    fi
}
#End Solaris Configuration Check
#########################################
#END Validate IP Configurations for Linux and Solaris and compare to ASR Manager
#########################################
#Begin Solaris Script Validations
#########################################
#Validate Perl Location and update if needed
perl_location_validation() {
#Perl Solaris Value Location Variable
LOC_PERL="/usr/perl5/5.12/bin/perl"
#Pull Current Location and store
Which_Perl=`which perl`
#check to see if the perl command is symbolically linked
if [ -L $Which_Perl ]; then
    #Pull Link location
    CUR_VER_PERL=`ls -l $Which_Perl | /usr/bin/awk -F'>' '{ print $2 }'`
    #Validate Location to Variables
    if [ $CUR_VER_PERL == $LOC_PERL ]; then
        feedback_to_screen "$CUR_VER_PERL meets current standards"
    else
        #if pulls incorrect link add the correct link to the system
        #remove existing link
        feedback_to_screen "Removing Old Perl Symbolic Link"
        rm $Which_Perl
        #create the correct symbolic link
        feedback_to_screen "Adding in correct Perl Symbolic Link Location"
        ln -s $LOC_PERL $Which_Perl
    fi
fi
}
#End Solaris Script Validations
#########################################
#Linux Validations
#########################################
#Modperl Validation
#Exadata Image 11.2.3.1 upgrade check

#End Modperl Validation
#########################################
#End Linux Script Validations
#########################################
#Start BMC Validation
bmc_validation() {
$IPMILOC/ipmitool bmc info > /dev/null 2>&1
        if [ $? -ne 0 ]; then
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Please Contact Oracle Support for BMC Communication issues"
            screen_output "BMC Communication is needed for ASR to setup Trap destinations as well as run as expected"
            screen_output "Once corrected, re-run ASREXACHK"
            BMC_STATUS="1"
        else
            screen_output "BMC Communication is Currently running"
            ILOM_VERSION=`$IPMILOC/ipmitool sunoem cli 'version'|grep "^SP firmware [0-9]\..*"|sed 's/SP firmware //g'`
            BMC_STATUS="0"
        fi
}
#End BMC Validation
#########################################
#Start asrs.state file validation
asrs_validate() {
    #Check for asrs.state file
    if [ -f $EXADATA_MON/asrs.state ]; then
        line_count=`wc -l $EXADATA_MON/asrs.state | /usr/bin/awk '{ print $1 }'`
        if [ $line_count -gt "1" ]; then
            sed '/^$/d' asrs.state > asrs.bck
            mv -f asrs.bck asrs.state
        fi
    fi
}
#End asrs.state file validation
#########################################
#Common Component Functions Status checks
#########################################
#Bug 20182652
#Start DBMCli Status Check
dbos_validate_checks() {
    #Check for type=asr
    feedback_to_screen "Checking for Type of ASR"
    if [ "$DUAL_ASR_ENTRY" == "TRUE" ]; then
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "Type of 'ASR' is for use for Auto Service Request only"
        screen_output "If using Grid Control or other monitoring systems, please leave the 'type' off of the command"
        screen_output "Please follow My Oracle Support Knowledge Article 1451026.1"
        ASR_TEST_VALUE="False"
    else
        ASR_TEST_VALUE="True"
    fi

    if [ $TESTFUNCTION == "TRUE" ]; then
        feedback_to_screen ""
        feedback_to_screen "Validating ASR Configuration with Validate SNMP"
        if [ $ASR_SETUP_STATUS == "Configured" ]; then
            #Send test event from DB Node
            #Send test event from OS
            feedback_to_screen ""
            feedback_to_screen "DB OS Validation"
            dbmcli -e "alter dbserver validate snmp type=asr" > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                ETH0_IP=`grep -i "IPADDR" /etc/sysconfig/network-scripts/ifcfg-eth0 | /usr/bin/awk -F= '{ print $2 }'`
                screen_output "OS Test event sent to $ASR_IP"
                db_os_snmp_val
                ASR_TEST_VALUE_OS="True"
            else
                ASR_TEST_VALUE_OS="False"
            fi
            #Send test event from ILOM
            feedback_to_screen ""
            feedback_to_screen "DB ILOM Validation"
            if [ $BMC_STATUS -eq 0 ]; then
                if [ $DB_ASR_COUNT -eq "1" ]; then
                    ilom_send_test
                else
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "Unable to Validate SNMP Due to Unconfigured ASR Setup"
                    screen_output "Please check the errors provided in the Configuration Check"
                    ASR_TEST_VALUE="False"
                fi
            else
                screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                screen_output "Unable to validate the ILOM SNMP Notification due to BMC Driver issues"
                screen_output "Please contact Oracle Exadata Support Team for assistance in correcting the BMC Driver"
                ASR_TEST_VALUE="False"
            fi
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Validate SNMP Due to Unconfigured ASR Setup"
            screen_output "Please check the errors provided in the Configuration Check"
            screen_output "Please contact ASR Support for help in resolving configuration issue"
            ASR_TEST_VALUE="False"
        fi
    fi
}
#End DBMCli Status Check
#########################################
#Start Cell Status Check
cell_validate_checks() {
    #Check for type=asr
    feedback_to_screen "Checking for Type of ASR"
    if [ "$DUAL_ASR_ENTRY" == "TRUE" ]; then
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "Type of 'ASR' is for use for Auto Service Request only"
        screen_output "If using Grid Control or other monitoring systems, please leave the 'type' off of the command"
        screen_output "Please follow My Oracle Support Knowledge Article 1451026.1"
        ASR_TEST_VALUE="False"
    else
        ASR_TEST_VALUE="True"
    fi
    #BMC State Check
    feedback_to_screen ""
    feedback_to_screen "Checking BMC Status"
    bmc_validation
    #Check for prefaulted devices
    #Check to see if PM Function is set to False before running
    if [ "$PMFUNCTION" == "FALSE" ]; then
        feedback_to_screen ""
        feedback_to_screen "Checking for Pre-Existing Errors"
        #Disk Fault check
        feedback_to_screen ""
        feedback_to_screen "Checking for HDD Faults"
        disk_fault_Status
        #Flash Fault Check
        feedback_to_screen ""
        feedback_to_screen "Checking for Flashdrive Failures"
        flash_fault_Status
        #FMA ILOM Check
        feedback_to_screen ""
        feedback_to_screen "Checking for FMA Faults"
        if [ $BMC_STATUS -eq 0 ]; then
            ilom_fault_FMA_Status
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check for FMA Faults due to BMC Driver Issue"
        fi
        #ILOM Hardware Faults
        feedback_to_screen ""
        feedback_to_screen "Checking for ILOM Hardware Faults"
        if [ $BMC_STATUS -eq 0 ]; then
            ilom_fault_Status
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check for ILOM Hardware Faults due to BMC Driver Issue"
        fi
    fi
    #End PM Function check
    if [ $TESTFUNCTION == "TRUE" ]; then
        feedback_to_screen ""
        feedback_to_screen "Validating ASR Configuration with Validate SNMP"
        if [ $ASR_SETUP_STATUS == "Configured" ]; then
            #Send test event from Cell Node
            #Send test event from OS
            feedback_to_screen ""
            feedback_to_screen "Cell OS Validation"
            cellcli -e "alter cell validate snmp type=asr" > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                ETH0_IP=`grep -i "IPADDR" /etc/sysconfig/network-scripts/ifcfg-eth0 | /usr/bin/awk -F= '{ print $2 }'`
                screen_output "OS Test event sent to $ASR_IP from $ETH0_IP"
                fromip="$ETH0_IP"
                cell_os_snmp_val
                ASR_TEST_VALUE_OS="True"
            else
                ASR_TEST_VALUE_OS="False"
            fi
            #Send test event from ILOM
            feedback_to_screen ""
            feedback_to_screen "Cell ILOM Validation"
            if [ $BMC_STATUS -eq 0 ]; then
                if [ $CELL_ASR_COUNT -eq "1" ]; then
                    ilom_send_test
                else
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "Unable to Validate SNMP Due to Unconfigured ASR Setup"
                    screen_output "Please check the errors provided in the Configuration Check"
                    ASR_TEST_VALUE="False"
                fi
            else
                screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                screen_output "Unable to validate the ILOM SNMP Notification due to BMC Driver issues"
                screen_output "Please contact Oracle Exadata Support Team for assistance in correcting the BMC Driver"
                ASR_TEST_VALUE="False"
            fi
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Validate SNMP Due to Unconfigured ASR Setup"
            screen_output "Please check the errors provided in the Configuration Check"
            screen_output "Please contact ASR Support for help in resolving configuration issue"
            ASR_TEST_VALUE="False"
        fi
    fi
}
#End Cell Status Check
#########################################
#Onecommand setup check (Exadata)
#Start onecommand setup
ex_oncmd(){
    if [ -f $EXA_ONECOMMAND ]; then 
        ASR_ENB=`grep -i "ASRENABLED" $EXA_ONECOMMAND | /usr/bin/awk -F= '{ print $2 }'`
        if [ "$ASR_ENB" == "1" ]; then
            screen_output "Exadata was setup for ASR by OnceCommand"
        fi
    fi
}
#End onecommand setup
#########################################
#Fromip validation to be eth0
#Start Fromip validation
fromip_validation() {
    $EXA_COMMAND -get_snmp_subscribers -type asr | grep fromip > /dev/null 2>&1
    if [ $? -eq 0 ]; then
        fromip=`grep ":$ASR_PORT" $EXADATA_MON/asrs.state | awk '{ print $7 }'`
        #Check fromip equals IP
        if [[ $fromip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
            #Pull Linux MGMT Interface
            if [ $OSTYPE = "LINUX" ]; then
                ifconfig eth0 | grep $fromip > /dev/null 2>&1
                if  [ $? -eq 0 ]; then
                    screen_output "Fromip matches the Management Interface"
                else
                    result=$(ASR_Return_9)
                    feedback_to_screen "$result"
                    ASR_SNMP="ASR-8001-20"
                    ASR_SNMP_DESC="ASR_Return_9"
                fi
            fi
            #Pull Solaris MGMT
            if [ $OSTYPE = "SOLARIS" ]; then
                #Pull Solaris eth0/net0 Interface IP
                ipadm show-if | grep -i "net0" > /dev/null 2>&1
                if [ $? -eq 0 ]; then
                    MGMTINT=`ipadm show-addr | grep -i "net0" | grep "v4" | /usr/bin/awk '{print $1}'`
                else
                    ipadm show-if | grep -i "eth0" > /dev/null 2>&1
                    if [ $? -eq 0 ]; then
                        MGMTINT=`ipadm show-addr | grep -i "eth0" | grep -v "bond" | grep -v "::" | /usr/bin/awk '{ print $4 }' | /usr/bin/awk -F/ '{print $1}'`
                    else
                        result=$(ASR_Return_5)
                        feedback_to_screen "$result"
                        ASR_SNMP="ASR-8001-HI"
                        ASR_SNMP_DESC="ASR_Return_5"
                    fi
                fi
                #Check MGMT to from IP
                if  [ $MGMTINT = $fromip ]; then
                    screen_output "Fromip matches the Management Interface"
                else
                    result=$(ASR_Return_9)
                    feedback_to_screen "$result"
                    ASR_SNMP="ASR-8001-20"
                    ASR_SNMP_DESC="ASR_Return_9"
                fi
            fi
        else
            #Hostname value for from IP
            result=$(ASR_Return_10)
            feedback_to_screen "$result"
            ASR_SNMP="ASR-8001-09"
            ASR_SNMP_DESC="ASR_Return_10"
        fi
    else
        #FROM IP not used
        result=$(ASR_Return_11)
        screen_output "$result"
    fi
}
#End fromip validation
#########################################
#End Common Component Functions Status checks
#########################################
# Proactive PM Function
#########################################
# PM Written by: Bob Belderbos, Michael M Johnson
#########################################
#Start Functions for PM

if [ $METHOD_TYPE == "Exadata" ] || [ $METHOD_TYPE == "Exalogic" ]; then #bug 20138670 // added exalogic as needed for some checks
    # general disk commands (defined here as used in various places)
    PD_LIST=`$MEGACLI PDList -aALL | grep "Firmware state"`
    NUM_DISKS=`echo "$PD_LIST"| wc -l`
    # firmware state should check for "Online, Spun Up" or "Hotspare"  either spun up or spun down
    NUM_DISKS_ONLINE=`echo "$PD_LIST"| egrep "Online, Spun Up|Hotspare, Spun" | wc -l` 

    # flash card uptime thresholds (cell check only)
    F20_UPTIME_THRESHOLD_HOURS=$((4*365*24)) # 4 years default, unless V2
    if [[ $SERVER_TYPE =~ .*X4275.* ]]; then
      F20_UPTIME_THRESHOLD_HOURS=$((3*365*24)) # 3 years 
    fi
fi #end bug 20138670


# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
# FUNCTION_DEFINITIONS
# <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

# A. reporting methods

#######################################
# Purpose: writes colored status message to output file, note that WARNINGS are going to both screen and file
# Globals: PASS, WARNING, NO_COLOR
# Arguments: $1 = status (ok, fail), $2 = message, $3 = output file
# Returns: None
#######################################
function status_msg {
  if [ "$1" == "PASS" ]; then 
    echo -e "\n${PASS}[PASS]${NO_COLOR} ${2}\n\n" >> $3
  else
    warn_string="\n${WARNING}[WARNING]${NO_COLOR} ${2}\n"
    echo -e "$warn_string" >> $3
    feedback_to_screen "$warn_string" # also write it to the script.out (should mirror what is printed on the screen)
  fi
}

#######################################
# Purpose: print a generic title / description before a check
# Globals: None
# Arguments: $1 = string to write, $2 = output file
# Returns: None
#######################################
function title_of_check {
  echo -e "\n** $1" >> $2
  printf '%100s\n' | tr ' ' - >> $2
}


# B. parsing methods

#######################################
# Purpose: checks for the right number of virtual disks
# Globals: OSTYPE, DB_NUM_VDISKS, DB_NUM_PDISKS
# Arguments: None
# Returns: 0 (true) or 1 (false)
#######################################
function db_vdisks_right_amount {
  # Virtual:  1 device if Linux,  4 for Solaris
  if [ "$OSTYPE" == "LINUX" ]; then
    if [ "$DB_NUM_VDISKS" -eq "1" ]; then
      return 0
    fi
  else 
    if [ "$DB_NUM_VDISKS" -eq "4" ]; then
      return 0
    fi
  fi
  return 1
}

#######################################
# Purpose: checks if no virtual disks are in degraded or offline mode
# Globals: DC_CONFIG
# Arguments: None
# Returns: 0 (true) or 1 (false)
#######################################
function db_vdisks_all_ok {
  num_degraded=`echo "$DC_CONFIG" | grep Degraded | sed 's/.*: \([0-9]*\)/\1/g'`
  num_offline=`echo "$DC_CONFIG" | grep Offline | sed 's/.*: \([0-9]*\)/\1/g'`
  if [ "$num_degraded" -eq "0" -a "$num_offline" -eq "0" ]; then
    return 0
  else
    return 1
  fi
}

#######################################
# Purpose: checks for the right number of physical disks
# Globals: SERVER_TYPE, DB_NUM_DISKS, DB_NUM_PDISKS
# Arguments: None
# Returns: 0 (true) or 1 (false)
#######################################
function db_pdisks_right_amount {
  if [ $METHOD_TYPE == "Exalogic" ]; then
    if [ "$DB_NUM_DISKS" -eq "2" ]; then
      return 0
    else
      return 1
    fi
  else
    # V2, X2-2, x3-2, X4-2 (X4170, X4170M2, X4170M3, X4-2) = 4 disks, 5 devices 
    # X2-8, X3-8 (X4800, X4800M2) = 8 disks, 11 devices
    # X4-8 (X4-8) = 7 disks, unknown devices (needs a system to check)
    if [[ $SERVER_TYPE =~ .*X4170.* ||  $SERVER_TYPE =~ .*X4-2.* ]]; then
      if [ "$DB_NUM_DISKS" -eq "4" -a "$DB_NUM_PDISKS" -eq "5" ]; then
        return 0
      fi
    elif [[ $SERVER_TYPE =~ .*X4800.* ]]; then
      if [ "$DB_NUM_DISKS" -eq "8" -a "$DB_NUM_PDISKS" -eq "11" ]; then
        return 0
      fi
    elif [[ $SERVER_TYPE =~ .*X4-8.* ]]; then
      if [ "$DB_NUM_DISKS" -eq "7" ]; then # $DB_NUM_PDISKS = unknown devices
        return 0
      fi
    fi
    return 1
  fi
}

#######################################
# Purpose: checks if no physical disks are critical or failed
# Globals: DC_CONFIG
# Arguments: None
# Returns: 0 (true) or 1 (false)
#######################################
function db_pdisks_all_ok {
  num_critical=`echo "$DC_CONFIG" | grep Critical | sed 's/.*: \([0-9]*\)/\1/g'`
  num_failed=`echo "$DC_CONFIG" | grep Failed | sed 's/.*: \([0-9]*\)/\1/g'`
  if [ "$num_critical" -eq "0" -a "$num_failed" -eq "0" ]; then
    return 0
  else
    return 1
  fi
}

#######################################
# Purpose: checks for the right number of physical disks on DB nodes
# Globals: SERVER_TYPE, NUM_DISKS_ONLINE
# Arguments: None
# Returns: 0 (true) or 1 (false)
#######################################
function right_num_phys_disks_online_dbnode {
  if [ $METHOD_TYPE == "Exalogic" ]; then
    if [ "$NUM_DISKS_ONLINE" -eq "2" ]; then
      return 0
    else
      return 1
    fi
  else
    if [[ $SERVER_TYPE =~ .*X4170.* || $SERVER_TYPE =~ .*X4-2.* ]]; then
      if [ "$NUM_DISKS_ONLINE" -eq "4" ]; then
        return 0
      fi
    elif [[ $SERVER_TYPE =~ .*X4800.* ]]; then
      if [ "$NUM_DISKS_ONLINE" -eq "8" ]; then
        return 0
      fi
    elif [[ $SERVER_TYPE =~ .*X4-8.* ]]; then
      if [ "$NUM_DISKS_ONLINE" -eq "7" ]; then
        return 0
      fi
    fi
    return 1
  fi
}

#######################################
# Purpose: determine correct flash cache size
# Globals: SERVER_TYPE, FLASH_CACHE_SIZE_ROUNDED
# Arguments: None
# Returns: 0 (true) or 1 (false)
#######################################
function right_flash_cache_size {
  # F20/F20M2 cards (V2, X2-2, X2-8 == X4275, X4270M2): 364.75G
  # F40 cards (X3-2, X3-8 == X4270M3): 1488.75G
  # F80 cards (X3-8, X4-2, X4-8 == X4270M3, X4-2L): 2978.75G
  if [[ $SERVER_TYPE =~ .*X4275.* || $SERVER_TYPE =~ .*X4270-M2.* ]]; then
    if [ "$FLASH_CACHE_SIZE_ROUNDED" -ge "364" ]; then
      return 0
    fi
  elif [[ $SERVER_TYPE =~ .*X4270-M3.* ]]; then
    if [ "$FLASH_CACHE_SIZE_ROUNDED" -ge "1488" ]; then
      return 0
    fi
  elif [[ $SERVER_TYPE =~ .*X4270-M3.* || $SERVER_TYPE =~ .*X4-2L.* ]]; then
    if [ "$FLASH_CACHE_SIZE_ROUNDED" -ge "2978" ]; then
      return 0
    fi
  fi
  return 1
}

#######################################
# Purpose: checks uptime of f20 card (if ilom target responds) based on card model
# Globals: IPMILOC, OUT_FLASH, F20_UPTIME_THRESHOLD_HOURS
# Arguments: $1 = f20card location (e.g. /SYS/MB/RISER2/PCIE5/F20CARD)
# Returns: None
#######################################
function check_f20card_uptime {
  card=$1
  cardInfo=`$IPMILOC/ipmitool sunoem cli "show $card"`
  echo "$cardInfo" | grep "^show: Invalid target" > /dev/null 2>&1

  if [ $? -eq 0 ]; then
    echo -e "\nCard $card returned Invalid target" >> $OUT_FLASH
    return 1
  fi

  cardPartNumber=`echo "$cardInfo" | grep fru_part_number | sed 's/.* = \([0-9][0-9]*-[0-9][0-9]*\).*/\1/g'`
  cardUptime=`$IPMILOC/ipmitool sunoem cli "show $card/UPTIME"`
  cardUptimeHours=`echo "$cardUptime" | grep "value =.*Hours" | sed 's/.*value = \([0-9][0-9]*\)\..*/\1/g'` # dot is like comma, drop it and also numbers (decimals) after it

  echo -e "\n$cardUptime" >> $OUT_FLASH
  echo -e "\nPart: $cardPartNumber - uptime: $cardUptimeHours - threshold: $F20_UPTIME_THRESHOLD_HOURS" >> $OUT_FLASH

  if [[ $cardUptimeHours =~ .*N/A.* ]]; then
    if [[ $card =~ .*PCIE4.* ]]; then
      echo "- uptime has N/A on PCIE4, likely due to bug, skip check" >> $OUT_FLASH
    else
      status_msg WARNING "Uptime value of $card is N/A" $OUT_FLASH
    fi
    return 1
  fi

  if (($cardUptimeHours > $F20_UPTIME_THRESHOLD_HOURS)); then
    status_msg WARNING "Uptime $card greater than threshold" $OUT_FLASH
  else
    status_msg PASS "Uptime $card below threshold" $OUT_FLASH
  fi
  return 0
}


#######################
# 1.x release - B items
#######################

function b_storage_items {
  which  cellcli 2>&1 >/dev/null
  if [ $? -ne 0 ]; then
    echo "Cellcli command not found, cannot check storage items\n" >> $OUT_STORAGE
    return 1
  fi

  title_of_check "Number of Luns on cells" $OUT_STORAGE
  num_hds=`cellcli -e list lun where disktype=harddisk | wc -l`
  num_fds=`cellcli -e list lun where disktype=flashdisk | wc -l`
  echo "Number of harddisk luns: $num_hds" >> $OUT_STORAGE
  echo "Number of flashdisk luns: $num_fds" >> $OUT_STORAGE
  if [ $num_hds -eq 12 -a $num_fds -eq 16 ]; then
    status_msg PASS "All luns are available on all storage servers" $OUT_STORAGE
  else
    status_msg WARNING "Not all luns are available on all storage servers" $OUT_STORAGE
  fi

  title_of_check "LUN status on Storage server" $OUT_STORAGE
  list_lun=`cellcli -e list lun`
  echo "$list_lun" >> $OUT_STORAGE
  echo "$list_lun" | grep -v normal 2>&1 > /dev/null
  if [ $? -eq 1 ]; then # 1 = grep returns nothing
    status_msg PASS "Status is normal for all LUNs on all storage servers" $OUT_STORAGE
  else
    status_msg WARNING "Status is not normal for one or more LUNs on storage servers" $OUT_STORAGE
  fi

  title_of_check "Celld status on storage servers" $OUT_STORAGE
  attr_status=`cellcli -e list cell attributes cellsrvStatus,msStatus,rsStatus`
  echo "$attr_status" >> $OUT_STORAGE
  if [[ $attr_status =~ running.*running.*running ]]; then
    status_msg PASS "Celld daemon is running on all storage servers" $OUT_STORAGE
  else
    status_msg WARNING "Celld daemon is not running on all storage servers" $OUT_STORAGE
  fi

  title_of_check "Exadata celldisk predictive failures" $OUT_STORAGE
  cd_pred_failures=`cellcli -e list celldisk | grep -v normal`
  if [ $? -eq 1 ]; then
    status_msg PASS "No celldisk has a status other than normal" $OUT_STORAGE
  else 
    status_msg WARNING "One or more celldisks have a status other than normal" $OUT_STORAGE
    echo "$cd_pred_failures" >> $OUT_STORAGE
  fi

  title_of_check "Number of celldisks on cells" $OUT_STORAGE
  num_celldisk_hds=`cellcli -e list lun where disktype=harddisk | wc -l`
  num_celldisk_fds=`cellcli -e list lun where disktype=flashdisk | wc -l`
  echo "Number of celldisk harddisks: $num_celldisk_hds" >> $OUT_STORAGE
  echo "Number of celldisk flashdisks: $num_celldisk_fds" >> $OUT_STORAGE
  if [ $num_celldisk_hds -eq 12 -a $num_celldisk_fds -eq 16 ]; then
    status_msg PASS "All celldisks are available on all storage servers" $OUT_STORAGE
  else
    status_msg WARNING "Not all celldisks are available on all storage servers" $OUT_STORAGE
  fi

  title_of_check "Verify storage server metric CD_IO_ST_RQ" $OUT_STORAGE
  cd_io_perf=0
  cd_io_st_rq_out=`cellcli -e list metriccurrent CD_IO_ST_RQ`
  echo "$cd_io_st_rq_out" >> $OUT_STORAGE
  while read CD_IO_ST_RQ; 
  do 
    limit=100000 # flash disk
    if [[ $CD_IO_ST_RQ =~  CD_[0-9][0-9]_ ]]; then # CD_ = conventional disk, flash has FD_[0-9][0-9]_ as pattern
      limit=300000 # conventional disk
    fi
    mc=`echo "$CD_IO_ST_RQ" | sed -e 's/,//g' -e 's/.* \([0-9][0-9]*\) us\/request.*/\1/g'` 
    if [ $mc -gt $limit ]; then 
      cd_io_perf=1
    fi; 
  done < <( echo "$cd_io_st_rq_out")
  if [ $cd_io_perf -eq 0 ]; then
    status_msg PASS "No Storage Server conventional or flash disks have a performance problem" $OUT_STORAGE
  else
    status_msg WARNING "One ore more Storage Server conventional or flash disks have a performance problem" $OUT_STORAGE
  fi

  title_of_check "Griddisks status on storage server" $OUT_STORAGE
  list_gdisk=`cellcli -e list griddisk | grep -v active`
  if [ $? -eq 1 ]; then
    status_msg PASS "Status is active for all griddisks on all storage servers" $OUT_STORAGE
  else 
    status_msg WARNING "Status is not active for all griddisks on all storage servers" $OUT_STORAGE
    echo "$list_gdisk" >> $OUT_STORAGE
  fi

  title_of_check "Verify griddisk count matches across all storage servers where a given prefix name exists" $OUT_STORAGE
  cell_group=/opt/oracle.SupportTools/onecommand/cell_group
  if [ ! -f $cell_group ]; then
    echo -e "File $cell_group not found, cannot verify griddisk count\n\n" >> $OUT_STORAGE
  else
    gd_count_out=""
    for GD_PREFIX in `dcli -g $cell_group -l root "cellcli -e list griddisk attributes name" | cut -d" " -f2 | gawk -F "_CD_" '{print $1}' | sort -u`;
    do
      GD_PREFIX_RESULT=`dcli -g $cell_group -l root  "cellcli -e list griddisk where name like \'$GD_PREFIX\_.*\' | wc -l" | cut -d" " -f2 | sort -u | wc -l`;
      if [ $GD_PREFIX_RESULT = 1 ]; then
        gd_count_out+="$GD_PREFIX: SUCCESS\n"
      else
        gd_count_out+="$GD_PREFIX: FAILURE\n"
        gd_count_out+=`dcli -g $cell_group -l root  "cellcli -e list griddisk where name like \'$GD_PREFIX\_.*\' | wc -l"`"\n"
      fi
    done

    echo -e "$gd_count_out" >> $OUT_STORAGE
    if [[ $gd_count_out =~ .*FAILURE.* ]]; then
      status_msg WARNING "The griddisk count matches across all storage servers where a given prefix name exists" $OUT_STORAGE
    else
      status_msg PASS "The griddisk count does not match across all storage servers where a given prefix name exists" $OUT_STORAGE
    fi
  fi

  title_of_check "Verify there are no griddisks configured on flash memory devices" $OUT_STORAGE
  gd_configured_on_fmem=`cellcli -e "list griddisk where disktype=flashdisk"`
  echo "$gd_configured_on_fmem" >> $OUT_STORAGE
  num_gd_configured_on_fmem=`echo "$gd_configured_on_fmem" | grep -v ^$ | wc -l`
  if [ $num_gd_configured_on_fmem -eq 0 ]; then
    status_msg PASS "There are no griddisks configured on flash memory devices" $OUT_STORAGE
  else
    status_msg WARNING "There are one or more griddisks configured on flash memory devices" $OUT_STORAGE
  fi

  title_of_check "Verify total number of griddisks with a given prefix name is evenly divisible of celldisks" $OUT_STORAGE
  CELLDISK_COUNT=`cellcli -e "list celldisk where disktype=harddisk" | wc -l`
  cd_count_out=""
  for GD_SHORT_NAME in `cellcli -e "list griddisk attributes name" | awk -F "_CD_" '{print $1}' | sort -u | egrep -v "DBFS|SYSTEM"`; 
  do 
    GD_COUNT=`cellcli -e "list griddisk where name like '$GD_SHORT_NAME.*'" | wc -l` 
    if [ `expr $GD_COUNT % $CELLDISK_COUNT` = 0 ]; then 
      cd_count_out+="$GD_SHORT_NAME:  SUCCESS\n"
    else
      cd_count_out+="$GD_SHORT_NAME:  FAILURE\n"
      cd_count_out+=`cellcli -e "list griddisk attributes name where name like '$GD_SHORT_NAME\_.*'"`"\n"
    fi 
  done
  echo -e "$cd_count_out" >> $OUT_STORAGE
  if [[ $cd_count_out =~ .*FAILURE.* ]]; then
    status_msg WARNING "The total number of griddisks with a given prefix name is not evenly divisible by the number of celldisks" $OUT_STORAGE
  else
    status_msg PASS "The total number of griddisks with a given prefix name is evenly divisible by the number of celldisks" $OUT_STORAGE
  fi


  title_of_check "Verify total size of all griddisks fully utilizes celldisk capacity" $OUT_STORAGE
  gd_attr_size=`cellcli -e "list griddisk attributes size"`
  if [ `echo "$gd_attr_size" | grep -v "^$" | wc -l` -eq 0 ]; then
    echo -e "Cellcli -e list griddisk attributes size gives 0 lines of output, cannot perform this check\n" >> $OUT_STORAGE
  else
    total_size_gdisks=$(echo `echo "$gd_attr_size" | awk '{ SUM += $1} END { print SUM}'` `cellcli -e "list celldisk attributes size where disktype=harddisk" | awk '{ SUM += $1} END { print SUM}'` | awk '{ printf("%% %f\n", ($1/$2)*100) }')
    echo -e "$total_size_gdisks\n" >> $OUT_STORAGE
    if [[ $total_size_gdisks =~ .*99.99.* ]]; then
      status_msg PASS "The total size of all griddisks fully utilizes celldisk capacity" $OUT_STORAGE
    else
      status_msg WARNING "The total size of all griddisks does not fully utilize celldisk capacity" $OUT_STORAGE
    fi
  fi
}
#End Functions for PM
#########################################
#Start PM Function
pm_function() {
# X5-2 not yet supported
    # Bug 20352672
    # X5-2 not yet supported
    if [[ $SERVER_TYPE =~ .*X5-2.* ]]; then
        feedback_to_screen "Checks are not required for the Preventive Maintenance process for platform $SERVER_TYPE"
        exit 1
    fi

    # End Bug 20352672
    feedback_to_screen "Node type: $NODE_TYPE"
    feedback_to_screen "ILOM version: $ILOM_VERSION"

    feedback_to_screen "\n** passing checks will be omitted here, see full outputs @ $LOGDIR **\n"

    feedback_to_screen "Performing generic checks ..."

    if [ $METHOD_TYPE == "Exadata" ]; then
      if [ "$NODE_TYPE" == "CELL" ]; then
        if [[ $SERVER_TYPE =~ .*X4275.* || $SERVER_TYPE =~ .*X4270-M2.* ]]; then 
          title_of_check "Verify PCI bridge is configured for generation II on storage servers" $OUT_GENERIC
          PCI_BRIDGES=`for BUS_NUM in 19:0.0 27:0.0; do echo $BUS_NUM $(lspci -xxx -s $BUS_NUM | grep ^50 | cut -d' ' -f4); done`
          echo "$PCI_BRIDGES" >> $OUT_GENERIC
          if [[ $PCI_BRIDGES =~ .*19:0.0.*82.* && $PCI_BRIDGES =~ .*27:0.0.*82.* ]]; then
            status_msg PASS "PCI bridges return value 82" $OUT_GENERIC
          else
            status_msg WARNING "One or more PCI bridges did not return value 82" $OUT_GENERIC
          fi
        fi
      fi
    fi

    # both exadata / exalogic 
    title_of_check "Verify Hardware and Firmware on Database and Storage Servers (CheckHWnFWProfile)" $OUT_GENERIC
    CHECK_HW_FW_SCRIPT=$EXADATA_MON/CheckHWnFWProfile
    if [ -f $CHECK_HW_FW_SCRIPT ]; then
        CHECK_HW=`$CHECK_HW_FW_SCRIPT` 
        echo "$CHECK_HW" >> $OUT_GENERIC
        if [ $METHOD_TYPE == "Exalogic" ]; then
          CHECK_HW_FW_SCRIPT_SUCCESS_STRING="Firmware verification succeeded"
        else
          CHECK_HW_FW_SCRIPT_SUCCESS_STRING="SUCCESS"
        fi
        if [[ $CHECK_HW =~ .*$CHECK_HW_FW_SCRIPT_SUCCESS_STRING.* ]]; then
          status_msg PASS "CheckHWnFWProfile found no issue" $OUT_GENERIC
        else
          status_msg WARNING "CheckHWnFWProfile found an issue" $OUT_GENERIC
        fi
    else
      echo "script $EXADATA_MON/CheckHWnFWProfile not found" >> $OUT_GENERIC
    fi

    if [ $METHOD_TYPE == "Exadata" ]; then
      if [ "$NODE_TYPE" == "CELL" ]; then
        title_of_check "Suitable USB device on storage servers" $OUT_GENERIC
        USB_DEV=`fdisk -l $(echo "$IMAGEINFO"| grep 'usb partition' | cut -d: -f2) 2>/dev/null | grep "^Disk.*MB.*bytes"`
        echo "$USB_DEV" >> $OUT_GENERIC
        echo "$USB_DEV" | grep ^Disk > /dev/null 2>&1
        if [ $? -ne 0 ]; then # if no output from fdisk -l ... 
          status_msg WARNING "USB drive is missing or not present" $OUT_GENERIC
        else
          # code to identify if fdisk USB is 8GB on X4 or earlier models (for X5 it is 8G)
          if [[ $SERVER_TYPE =~ .*X4275.* || $SERVER_TYPE =~ .*X4270-M[23].* || $SERVER_TYPE =~ .*X4-2L.*  ]]; then
            if [[ $USB_DEV =~ ^Disk.*/dev.*4[0-9][0-9][0-9].*MB.* ]]; then
              status_msg PASS "Size of USB drive is 4G" $OUT_GENERIC
            else
              status_msg WARNING "Size of USB drive is not 4G" $OUT_GENERIC
            fi
          fi
        fi

        title_of_check "Alerthistory log" $OUT_GENERIC
        cellcli -e list alerthistory >> $OUT_GENERIC

        title_of_check "Image partition status on storage servers" $OUT_GENERIC
        echo "$IMAGEINFO" >> $OUT_GENERIC
      fi
    fi


    # both exadata / exalogic 
    feedback_to_screen "\nChecking for FMA faults and collecting ILOM outputs ..."

    title_of_check "Open FMA faults" $OUT_ILOM
    FMA_FAULTY=`$IPMILOC/ipmitool sunoem cli 'show faulty'`
    echo "$FMA_FAULTY" >> $OUT_ILOM
    echo "$FMA_FAULTY" | grep ^/SP/faultmgmt > /dev/null 2>&1
    if [ $? -ne 0 ]; then
      status_msg PASS "No active FMA faults" $OUT_ILOM
    else
      status_msg WARNING "One or more active FMA faults" $OUT_ILOM
    fi

    # ilom properties
    if [[ $ILOM_VERSION =~ .*3\.0\..* ]]; then # no more ilom 2
      title_of_check "ILOM version < 3.1, collecting show /SYS and show /SP" $OUT_ILOM
      $IPMILOC/ipmitool sunoem cli 'show /SYS' >> $OUT_ILOM
      $IPMILOC/ipmitool sunoem cli 'show /SP' >> $OUT_ILOM
    else
      title_of_check "ILOM version 3.1 or higher, collecting show /System" $OUT_ILOM
      $IPMILOC/ipmitool sunoem cli 'show /System' >> $OUT_ILOM
    fi

    title_of_check "Get status LEDs" $OUT_ILOM
    $IPMILOC/ipmitool sunoem led get all >> $OUT_ILOM


    # both exadata / exalogic 
    feedback_to_screen "\nChecking battery and disk status ..."

    title_of_check "Verify RAID Controller Battery Condition" $OUT_DISKS
    BBU_CMD_OUT=`$MEGACLI -AdpBbuCmd -a0`
    # does this command get valid output?
    echo "$BBU_CMD_OUT"|grep "Adapter.*Get BBU Status Failed" > /dev/null 2>&1
    if [ $? -eq 0 ]; then
      echo "$MEGACLI -AdpBbuCmd -a0 returned Adapter.*Get BBU Status Failed, cannot check batteries" >> $OUT_DISKS
    else
      BAT_COND_OUT=`echo "$BBU_CMD_OUT" | egrep "Full Charge|Max Error|BatteryType" | sort | head -3`
      echo "$BAT_COND_OUT" >> $OUT_DISKS
      BAT_MAH=`echo "$BAT_COND_OUT" | grep "Full Charge" | sed 's/.*: \([0-9]*\) mAh/\1/g'`
      BAT_MAX_ERR=`echo "$BAT_COND_OUT" | grep "Max Error" | sed 's/.*: \([0-9]*\) %/\1/g'`
      if [ "$BAT_MAH" -lt "800" -a "$BAT_MAX_ERR" -lt "10" ]; then
        status_msg WARNING "Proactive battery replacement should be performed within 60 days" $OUT_DISKS
      elif [ "$BAT_MAX_ERR" -ge "10" -o "$BAT_MAH" -lt "674" ]; then
        status_msg WARNING "Battery replacement should be performed" $OUT_DISKS
      else
        status_msg PASS "Battery status is OK" $OUT_DISKS
      fi

      title_of_check "Verify RAID Controller Battery Temperature" $OUT_DISKS
      BAT_TYPE_TEMP_OUT=`echo "$BBU_CMD_OUT" | egrep "BatteryType|Temper"`
      echo "$BAT_TYPE_TEMP_OUT" >> $OUT_DISKS
      BAT_TEMP=`echo "$BAT_TYPE_TEMP_OUT" | grep "^Temp.*C" | sed 's/.*: \([0-9]*\) C/\1/g'`
      if [ "$BAT_TEMP" -gt "55" ]; then
        status_msg WARNING "Battery temperature is greater than 55 C" $OUT_DISKS
      else
        status_msg PASS "Battery temperature is below 55 C" $OUT_DISKS
      fi
      echo "$BBU_CMD_OUT" | grep "Over Temperature" > /dev/null 2>&1 
      if [ $? -eq 0 ]; then
        echo "$BBU_CMD_OUT" | grep "Over Temperature *: No" > /dev/null 2>&1 
        if [ $? -eq 0 ]; then
          status_msg PASS "No Battery Over Temperature" $OUT_DISKS
        else
          status_msg WARNING "Battery Over Temperature" $OUT_DISKS
        fi
      fi
    fi


    if [ "$NODE_TYPE" == "CELL" ] || [ $METHOD_TYPE == "Exalogic" ]; then
      title_of_check "RAID controller version on storage servers" $OUT_DISKS
      $MEGACLI adpallinfo a0 | grep -i package >> $OUT_DISKS
    fi

    if [ "$NODE_TYPE" == "DB" ] || [ $METHOD_TYPE == "Exalogic" ]; then
        title_of_check "Verify Database Server Disk Controller configuration" $OUT_DISKS
        DC_CONFIG=`$MEGACLI AdpAllInfo -aALL | $GGREP "Device Present" -A8` 
        DB_NUM_VDISKS=`echo "$DC_CONFIG" | grep Virtual | sed 's/.*: \([0-9]*\)/\1/g'`
        DB_NUM_PDISKS=`echo "$DC_CONFIG" | grep Physical | sed 's/.*: \([0-9]*\)/\1/g'`
        DB_NUM_DISKS=`echo "$DC_CONFIG" | grep "^ *Disks" | sed 's/.*: \([0-9]*\)/\1/g'`
        echo "$DC_CONFIG" >> $OUT_DISKS

        # a. vdisks
        DB_CONFIG_OK=true
        if ! db_vdisks_right_amount; then
          status_msg WARNING "Wrong number of virtual disks" $OUT_DISKS
          DB_CONFIG_OK=false
        fi
        if ! db_vdisks_all_ok; then
          status_msg WARNING "One or more virtual disks degraded or offline" $OUT_DISKS
          DB_CONFIG_OK=false
        fi

        # b. phys disks
        if ! db_pdisks_right_amount; then
          status_msg WARNING "Wrong number of physical disks" $OUT_DISKS
          DB_CONFIG_OK=false
        fi
        if ! db_pdisks_all_ok; then
          status_msg WARNING "One or more physical disks critical or failed" $OUT_DISKS
          DB_CONFIG_OK=false
        fi

        if $DB_CONFIG_OK; then
          status_msg PASS "No issues found with disk controller configuration" $OUT_DISKS
        fi

      title_of_check "Verify Database Server Physical Drive Configuration" $OUT_DISKS
      echo "Number of disks online: $NUM_DISKS_ONLINE" >> $OUT_DISKS
      if ! right_num_phys_disks_online_dbnode; then
        status_msg WARNING "Wrong number of physical drives with Online, Spun Up (or Hotspare) state" $OUT_DISKS
      else
        status_msg PASS "Correct number of physical drives with Online, Spun Up (or Hotspare) state" $OUT_DISKS
      fi
    fi

    if [ $METHOD_TYPE == "Exadata" ]; then
      if [ "$NODE_TYPE" == "CELL" ]; then
        title_of_check "Physical disk status on cells" $OUT_DISKS
        echo "$PD_LIST" >> $OUT_DISKS
        echo "$PD_LIST" | egrep -v "Online, Spun Up|Hotspare, Spun" > /dev/null 2>&1 # if a disk is in another state grep -v returns 0 
        if [ $? -eq 0 ]; then
          status_msg WARNING "Not all disks are in Online, Spun Up (or Hotspare) state" $OUT_DISKS
        else
          status_msg PASS "All disks are in Online, Spun Up (or Hotspare) state" $OUT_DISKS
        fi

        title_of_check "Physicaldisk Count on storage servers" $OUT_DISKS
        if [ "$NUM_DISKS" -ne "12" ]; then 
          status_msg WARNING "Not 12 disks found for a storage node" $OUT_DISKS
        else
          status_msg PASS "12 disks found for a storage node" $OUT_DISKS
        fi
      fi
    fi


    # both exadata / exalogic 
    title_of_check "Verify Disk Cache Policy using writeback cache" $OUT_DISKS
    LD_PD_INFO=`$MEGACLI -LdPdInfo -aALL`
    DISK_CACHE_POLICY=`echo "$LD_PD_INFO" | grep -i "Disk Cache Policy"`
    echo "$DISK_CACHE_POLICY" >> $OUT_DISKS
    DISK_CACHE_POLICY_COUNT=`echo "$DISK_CACHE_POLICY" | grep -v "Disk.*Default" | grep -v Disabled | wc -l`
    if [ "$DISK_CACHE_POLICY_COUNT" -eq "0" ];  then
      status_msg PASS "Disk Cache Policy disabled on all disks" $OUT_DISKS
    else
      status_msg WARNING "Disk Cache Policy not disabled on all disks" $OUT_DISKS
    fi
    DISK_CACHE_DISK_DEFAULT=`echo "$DISK_CACHE_POLICY" | grep "Disk.*Default" | wc -l`
    if [ "$DISK_CACHE_DISK_DEFAULT" -gt "0" ];  then
      echo "INFO: disk default status detected" >> $OUT_DISKS
    fi

    # FLASH 
    if [ $METHOD_TYPE == "Exadata" ]; then
      if [ "$NODE_TYPE" == "CELL" ]; then
        feedback_to_screen "\nChecking flash devices ..."

        title_of_check "Faulted flash cards" $OUT_FLASH
        FLASH_CARDS=`cellcli -e list physicaldisk where disktype=flashdisk`
        echo "$FLASH_CARDS" >> $OUT_FLASH
        echo "$FLASH_CARDS" | grep -v normal > /dev/null 2>&1 # if flash card is in another state grep -v returns 0
        if [ $? -eq 0 ]; then
          status_msg WARNING "Not all flash cards are in normal state" $OUT_FLASH
        else
          status_msg PASS "All flash cards are in normal state" $OUT_FLASH
        fi

        title_of_check "Flash cache mode - WT or WB " $OUT_FLASH
        cellcli -e list cell attributes flashCacheMode >> $OUT_FLASH

        title_of_check "Verify Exadata Smart Flash Cache is full size" $OUT_FLASH
        FLASH_CACHE_SIZE=`cellcli -e "list flashcache detail" | grep size`
        FLASH_CACHE_SIZE_ROUNDED=`echo "$FLASH_CACHE_SIZE" | sed 's/.*size:.* \([0-9][0-9]*\)\.[0-9][0-9]*G/\1/g'`
        echo "$FLASH_CACHE_SIZE" >> $OUT_FLASH
        if ! right_flash_cache_size; then
          status_msg WARNING "Flash cache size is less than expected" $OUT_FLASH
        else
          status_msg PASS "Flash cache size is correct" $OUT_FLASH
        fi

      if [[ $SERVER_TYPE =~ .*X4275.* || $SERVER_TYPE =~ .*X4270-M2.* ]]; then
          title_of_check "Verify Electronic Storage Module (ESM) Lifetime is within Specification" $OUT_FLASH
          check_f20card_uptime /SYS/MB/RISER1/PCIE1/F20CARD
          check_f20card_uptime /SYS/MB/RISER1/PCIE4/F20CARD
          check_f20card_uptime /SYS/MB/RISER2/PCIE2/F20CARD
          check_f20card_uptime /SYS/MB/RISER2/PCIE5/F20CARD

        fi
      fi
    fi

    # 20764710 (including supercluster and zdlra)
    if [ "$NODE_TYPE" == "CELL" ]; then
      # F20/F20M2 cards only 
      if [[ $SERVER_TYPE =~ .*X4275.* || $SERVER_TYPE =~ .*X4270-M2.* ]]; then
        feedback_to_screen "\nChecking types of F20 cards installed to match ESM part numbers ..."
        title_of_check "Collecting flash_dom -l output, grepping for SAS1068E B3 (F20) / C0 (F20M2) cards, and printing corresponding ESM part info" $OUT_FLASH
        flash_dom -l | grep SAS1068E | sed -e 's/SAS1068E B3/SAS1068E B3 (F20 -> ESM 371-4650)/g' -e 's/SAS1068E C0/SAS1068E C0 (F20M2 -> ESM 371-4953)/g' >> $OUT_FLASH
      fi
    fi
    # end flash


    #Bug 20352753
    # call new function to run b items
    if [ $METHOD_TYPE == "Exadata" ]; then
      if [ "$NODE_TYPE" == "CELL" ]; then
        feedback_to_screen "\nPerforming storage server checks ..."
        b_storage_items
      fi
    fi
    
    # check for faults at end and if any, ask for a sundiag (with ilom option = snapshot included)
    feedback_to_screen "\n--------\nRESULT\n--------"
    WARNINGS=`grep -h ".*1;33m\[WARNING\]" $LOGDIR/*out 2>/dev/null`
    NUM_WARNINGS=`echo "$WARNINGS" | grep -v '^$' | wc -l`
    if [ "$NUM_WARNINGS" -eq "0" ]; then
      feedback_to_screen "\n${PASS}[PASS]${NO_COLOR} all checks passed successfully."
    else
      feedback_to_screen "\n${WARNING}[WARNING]${NO_COLOR} not all checks passed successfully.\n"
      feedback_to_screen "Please collect a sundiag with the snapshot option to further diagnose the system: "
      feedback_to_screen "# /opt/oracle.SupportTools/sundiag.sh snapshot\n"
      feedback_to_screen "See also: https://support.oracle.com/epmos/faces/DocumentDisplay?id=761868.1" #Fixed for Bug 20121784
    fi

    # include *log files
    RAIDDIR="${LOGDIR}/raid"
    mkdir $RAIDDIR
    MEGASAS=`pwd`/MegaSAS.log
    CMDTOOL=`pwd`/CmdTool.log
    if [ -f $MEGASAS ]; then
      mv $MEGASAS $RAIDDIR
    fi
    if [ -f $CMDTOOL ]; then
      mv $CMDTOOL $RAIDDIR
    fi
}
# End PM Function
#########################################
#Test Fault Injectors to send Test Event Emails and validate Path to ASR Manager
#########################################
# ILOM Test Event Function
ilom_send_test() {
    #Pull ILOM IP from BMC
    ILOM_IP=`$IPMILOC/ipmitool lan print | grep -i "IP Address              :" | /usr/bin/awk -F: '{print $2 }' | sed -e 's/^[ \t]*//'`
    #Pull ASR Manager IP to validate for test event
    #Pull Node Type
    if [ -f $EXA_COMMAND ]; then
        ASR_MAN_IP_DB
    else
        if [ -f /opt/oracle.cellos/ORACLE_CELL_NODE ]; then
            ASR_MAN_IP_CELL
        fi
    fi
    #Check which rules location has the ASR Manager IP
        for i in {1..15}
            do
                RULES=`$IPMILOC/ipmitool sunoem cli "show /SP/alertmgmt/rules/$i destination"`
                echo $RULES | grep "$ASR_IP" > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                RULE_VER=`echo $i`
                break
            fi
        done
    #Create Test Event
    if [ -n "$RULE_VER" ]; then
    SNMP_RULES=`$IPMILOC/ipmitool sunoem cli "show /SP/alertmgmt/rules/$RULE_VER snmp_version"`
        echo $SNMP_RULES | grep "2c" > /dev/null 2>&1
        if [ $? -ne 0 ]; then
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "The SNMP Version is set to an unsupported type."
            screen_output "Please rerun the ASR setup methods to have the snmp"
            screen_output "version to the correct method."
        fi
    RULE_TEST=`$IPMILOC/ipmitool sunoem cli "set /SP/alertmgmt/rules/$RULE_VER testrule=true"`
        echo $RULE_TEST | grep "To send a test alert" > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to send Test Event due to ILOM Configuration issue"
            screen_output "Please check the ILOM Configuration to make sure the values are set correctly per ASR Exadata Scripts"
            ASR_TEST_VALUE_ILOM="False"
        else
            screen_output "ILOM Test event sent to $ASR_IP:$ASR_PORT from $ILOM_IP"
            ASR_TEST_VALUE_ILOM="True"
        fi
    else
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "Unable to send Test Event due to ILOM Configuration issue"
        screen_output "Please check the ILOM Configuration to make sure the values are set correctly per ASR Exadata Scripts"
        ASR_TEST_VALUE_ILOM="False"
    fi
}
#END ILOM Test Event Function
#########################################
#Bug 20182652
#DB OS fault SNMP Validation check 
#Validation that MS Service has the systemidentifier value
#Start SNMP validation
db_os_snmp_val() {
HOST=`hostname | /usr/bin/awk -F. '{ print $1 }'`
MS_ODL="/var/log/oracle/diag/asm/dbserver/$HOST/trace/ms-odl.log"
Date=`date '+%Y-%m-%dT%H:%M'`
grep -i "ASR SNMP" "$MS_ODL" | grep $Date | grep test | egrep "$DEV_TYPE|Exadata Storage" > /dev/null 2>&1
if [ $? -ne 0 ]; then
    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
    screen_output "ASRExachk has determined that there was a failure during the SNMP Trap Verification."
    screen_output "The failure will not inhibit the ability for ASR to create a SR but can cause delay"
    screen_output "Diagnostic time due to System Identifier missing."
    screen_output "To fix this error, please open a Service Request with the Oracle Exadata Team"
    echo `date` "Missing SystemIdentifier in SNMP Configuration" >> $LOG
fi
}
#End SNMP Validation
#END Test Fault Injectors to send Test Event Emails and validate Path to ASR Manager
#########################################
#Cell OS fault SNMP Validation check
#Validation that MS Service has the systemidentifier value
#Start SNMP validation
cell_os_snmp_val() {
HOST=`hostname | /usr/bin/awk -F. '{ print $1 }'`
MS_ODL="/var/log/oracle/diag/asm/cell/$HOST/trace/ms-odl.log"
Date=`date '+%Y-%m-%dT%H:%M'`
grep -i "ASR SNMP" "$MS_ODL" | grep $Date | grep test | egrep "$DEV_TYPE|Exadata Storage" > /dev/null 2>&1
if [ $? -ne 0 ]; then
    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
    screen_output "ASRExachk has determined that there was a failure during the SNMP Trap Verification."
    screen_output "The failure will not inhibit the ability for ASR to create a SR but can cause delay"
    screen_output "Diagnostic time due to System Identifier missing."
    screen_output "To fix this error, please open a Service Request with the Oracle Exadata Team"
    echo `date` "Missing SystemIdentifier in SNMP Configuration" >> $LOG
fi
}
#End SNMP Validation
#END Test Fault Injectors to send Test Event Emails and validate Path to ASR Manager
#########################################
#Check for existing Faults
#########################################
#Disk Fault Check
disk_fault_Status() {
    #Check to see if temp file was not removed previously
    if [ -f /tmp/ASR-HDD-Fault-Check-Log ]; then
        rm -f /tmp/ASR-HDD-Fault-Check-Log
    fi
    #Check for Faulted drives
    HDD_FAULTED=`$MEGACLI -pdlist -aALL | awk '/Slot Number/ { counter += 1; slot[counter] = $3 } /Device Id/ { device[counter] = $3 } /Firmware state/ { state_drive[counter] = $3 } /Inquiry/ { name_drive[counter] = $3 " " $4 " " $5 " " $6 } END { for (i=1; i<=counter; i+=1) printf ( "Slot %02d Device %02d (%s) status is: %s \n", slot[i], device[i], name_drive[i], state_drive[i]); }' | grep -i -v Online | grep -i -v Hotspare`
    #If returns faulted drive, report the faulted values
    if [ $? -eq 0 ]; then
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        echo $HDD_FAULTED >> /tmp/ASR-HDD-Fault-Check-Log
        screen_output "Hardware Error has been detected in the HDD Logs on the Cell Node"
        screen_output "Please open up a Service Request with Oracle to resolve the Error"
        screen_output "View File /tmp/ASR-HDD-Fault-Check-Log for further"
        screen_output "Information of the Error. This log File can be copied to the "
        screen_output "Oracle Service Request for Error Reference"
    else
        screen_output "No HDD Faults Found"
    fi
}
#END Disk Fault Check
#########################################
#Flash Disk Fault Check Cell Only
flash_fault_Status() {
    #Check to see if temp file was not removed previously
    if [ -f /tmp/ASR-Flash-Fault-Check-Log ]; then
        rm -f /tmp/ASR-Flash-Fault-Check-Log
    fi
    #Flash Drive failure Check Cell only
    FLASH_FAULT=`cellcli -e list physicaldisk where status!=normal | grep FLASH`
    #If returns faulted value, report the faulted values
    if [ $? -eq 1 ]; then
        screen_output "No Flash Faults found"
    else
        echo $FLASH_FAULT >> /tmp/ASR-Flash-Fault-Check-Log
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "Hardware Error has been detected in the Flash Logs on the Cell Node"
        screen_output "Please open up a Service Request with Oracle to resolve the Error"
        screen_output "View File /tmp/ASR-Flash-Fault-Check-Log for further"
        screen_output "Information of the Error. This log File can be copied to the "
        screen_output "Oracle Service Request for Error Reference"
    fi
}
#END Flash Disk Fault Check Cell Only
#########################################
#FMA Fault Check Cell Only
ilom_fault_FMA_Status() {
    #Check to see if temp file was not removed previously
    if [ -f /tmp/ASR-FMA-Fault-Check.log ]; then
        rm -f /tmp/ASR-FMA-Fault-Check.log
    fi
    #ILOM Faults
    echo `$IPMILOC/ipmitool sunoem cli "start /SP/faultmgmt/shell" "y" "fmadm faulty -a"` >> /tmp/ASR-FMA-Fault-Check.log
    if [ $? -eq 0 ]; then
        if [ `wc -l /tmp/ASR-FMA-Fault-Check.log|cut -d' ' -f1` -lt 2 ]; then
          screen_output "No output from fmadm faulty -a"
        else
          grep "No faults found" /tmp/ASR-FMA-Fault-Check.log > /dev/null 2>&1
          #If returns faulted value, report the faulted values
          if [ $? -eq 0 ]; then
              screen_output "No FMA faults found"
              rm -f /tmp/ASR-FMA-Fault-Check.log
          else
              screen_output "${WARNING}#########Warning#########${NO_COLOR}"
              screen_output "Hardware Error has been detected in the FMA Logs on the ILOM"
              screen_output "Please open up a Service Request with Oracle to resolve the Error"
              screen_output "View File /tmp/ASR-FMA-Fault-Check.log for further "
              screen_output "Information of the Error. This log File can be copied to the Oracle"
              screen_output "Service Request for Error Reference"
          fi
        fi
    else
        screen_output "The ILOM is reporting busy, please run script again at a later time"
    fi
}
#END FMA Fault Check Cell Only
#########################################
#ILOM Hardware Fault Check Cell Only
ilom_fault_Status() {
    #Check to see if temp file was not removed previously
    if [ -f /tmp/ASR-ILOM-Fault-Check.log ]; then
        rm -f /tmp/ASR-ILOM-Fault-Check.log
    fi
    #Pull Hardware Faults
    $IPMILOC/ipmitool sunoem cli "show faulty" | /usr/bin/awk '/-----/,/->/' |  grep -v "+-" | grep -v "Session closed" >> /tmp/ASR-ILOM-Fault-Check.log
    if [ $? -eq 0 ]; then
        grep -i "sunw-msg-id" /tmp/ASR-ILOM-Fault-Check.log > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            #If returns faulted value, report the faulted values
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Hardware Error has been detected in the ILOM Logs"
            screen_output "Please open up a Service Request with Oracle to resolve the Error"
            screen_output "View File /tmp/ASR-ILOM-Fault-Check.log for further"
            screen_output "Information of the Error. This log File can be copied to the Oracle"
            screen_output "Service Request for Error Reference"
        else
            screen_output "No Hardware ILOM Faults"
            rm -f /tmp/ASR-ILOM-Fault-Check.log
        fi
    else
        screen_output "The ILOM is reporting busy, please run script again at a later time"
    fi
}
#END ILOM Hardware Fault Check Cell Only
#########################################
#END Check for existing Faults
#########################################
#Main Functions for status checks
#########################################

function show_cached_asrs_state() {
  feedback_to_screen "\nNote that this $METHOD_TYPE node has been previously setup with asrexachk"
  feedback_to_screen "\tCached file $EXADATA_MON/asrs.state contains: "
  feedback_to_screen "\t- ASR Manager IP: `cut -d":" -f1 $EXADATA_MON/asrs.state`"
  feedback_to_screen "\t- ASR Manager SNMP Port: `cut -d":" -f2 $EXADATA_MON/asrs.state`"
  feedback_to_screen "** If you wish to change these values, delete this file and re-run this script ...\n"
}


#########################################
#Status Start location
#########################################
#Return Host Name value
feedback_to_screen "$VERSION"
feedback_to_screen "`date`"
feedback_to_screen "Hostname: $HOSTNAME"

#Print out Device Type
feedback_to_screen "Device Type: $DEV_TYPE"


feedback_to_screen "Serial Number: $SERIAL_NUM"
feedback_to_screen "Server Type: $SERVER_TYPE"
feedback_to_screen "OS Type: $OSTYPE"
feedback_to_screen "SNMP Port: $ASR_PORT"

echo `date` $VERSION >> $LOG

#########################################
#Start Exadata Methods
if [ $METHOD_TYPE == "Exadata" ]; then
    #########################################
    #Node type validation
    if [ "$IMAGEINFO_NODE" == "STORAGE" ]; then
        NODE_TYPE="CELL"

        
        #Cellcli retrieve trap destination of type ASR
        cellcli -e "list cell attributes snmpsubscriber" | grep -i "type=asr" > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            cellcli -e list cell detail | grep -i notificationMethod | grep -i snmp > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                ASR_SETUP_STATUS="Configured"
                ex_oncmd
                ASR_MAN_IP_CELL
                feedback_to_screen "ASR Manager IP: $ASR_IP"
                CELL_LIST=`cellcli -e "list cell attributes snmpsubscriber" | sed -e 's/^[ \t]*//'`
                feedback_to_screen "SNMP Trap Destinations: $CELL_LIST"
                feedback_to_screen "Exadata Cell Image: $EXA_CELL_IMAGE"
            else
                cellcli -e list cell detail | grep -i notificationMethod | grep -i mail > /dev/null 2>&1
                if [ $? -eq 0 ]; then
                    feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                    feedback_to_screen "ASREXACHK has found a configuration issue with the Cell Notification Method."
                    feedback_to_screen "ASREXACHK returned value of 'mail' only"
                    feedback_to_screen "Running command to set value to include SNMP"
                    cellcli -e "alter cell notificationMethod='mail,snmp'" > /dev/null 2>&1
                    ASR_SETUP_STATUS="Configured"
                else
                    feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                    feedback_to_screen "ASREXACHK has found a configuration issue with the Cell Notification Method."
                    feedback_to_screen "ASREXACHK returned value with no method set"
                    feedback_to_screen "Running command to set value to include SNMP"
                    cellcli -e "alter cell notificationMethod='snmp'" > /dev/null 2>&1
                    ASR_SETUP_STATUS="Configured"
                fi
            fi
        else
            asr_not_configured 
            ASR_SETUP_STATUS="Unconfigured"
            feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
            feedback_to_screen "Unable to determine configuration status of the cell node."
            feedback_to_screen "Please check to make sure the ASR Manager Hostname or IP are"
            feedback_to_screen "setup through the cellcli command to include the type of ASR"
            feedback_to_screen "Please reference the Oracle Exadata Administration guide for more"
            feedback_to_screen "details on cell commands for ASR as well"
            feedback_to_screen "as My Oracle Support Knowledge Article 1451026.1 for use with Grid Control"
        fi
    elif [ "$IMAGEINFO_NODE" == "COMPUTE" ]; then
        NODE_TYPE="DB"
        #Bug 20182652
        if [ "$DBMCLI" == "TRUE" ]; then
            #Cellcli retrieve trap destination of type ASR
            dbmcli -e "list dbserver attributes snmpsubscriber" | grep -i "type=asr" > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                dbmcli -e list dbserver detail | grep -i notificationMethod | grep -i snmp > /dev/null 2>&1
                if [ $? -eq 0 ]; then
                    ASR_SETUP_STATUS="Configured"
                    ex_oncmd
                    ASR_MAN_IP_DB
                    feedback_to_screen "ASR Manager IP: $ASR_IP"
                    DB_LIST=`dbmcli -e "list dbserver attributes snmpsubscriber" | sed -e 's/^[ \t]*//'`
                    feedback_to_screen "SNMP Trap Destinations: $DB_LIST"
                    feedback_to_screen "Exadata Node Image: $EXA_DB_IMAGE"
                else
                    dbmcli -e list dbserver detail | grep -i notificationMethod | grep -i mail > /dev/null 2>&1
                    if [ $? -eq 0 ]; then
                        feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                        feedback_to_screen "ASREXACHK has found a configuration issue with the DB Notification Method."
                        feedback_to_screen "ASREXACHK returned value of 'mail' only"
                        feedback_to_screen "Running command to set value to include SNMP"
                        dbmcli -e "alter dbserver notificationMethod='mail,snmp'" > /dev/null 2>&1
                        ASR_SETUP_STATUS="Configured"
                    else
                        feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                        feedback_to_screen "ASREXACHK has found a configuration issue with the DB Notification Method."
                        feedback_to_screen "ASREXACHK returned value with no method set"
                        feedback_to_screen "Running command to set value to include SNMP"
                        dbmcli -e "alter dbserver notificationMethod='snmp'" > /dev/null 2>&1
                        ASR_SETUP_STATUS="Configured"
                    fi
                fi
            else
                asr_not_configured 
                ASR_SETUP_STATUS="Unconfigured"
                feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                feedback_to_screen "Unable to determine configuration status of the db node."
                feedback_to_screen "Please check to make sure the ASR Manager Hostname or IP are"
                feedback_to_screen "setup through the dbmcli command to include the type of ASR"
                feedback_to_screen "Please reference the Oracle Exadata Administration guide for more"
                feedback_to_screen "details on db commands for ASR as well"
                feedback_to_screen "as My Oracle Support Knowledge Article 1451026.1 for use with Grid Control"
            fi
        else
            if [ -f $EXADATA_MON/asrs.state ]; then
                if [ $? -eq 0 ]; then
                    ASR_SETUP_STATUS="Configured"
                    asrs_validate
                    ex_oncmd
                    ASR_MAN_IP_DB
                    if [ "$ASR_IP" != "N/A" ]; then
                        feedback_to_screen "ASR Manager IP: $ASR_IP"
                        SNMP_TRAP_DEST=`$EXA_COMMAND -get_snmp_subscribers -type asr | sed -e 's/^[ \t]*//'`
                        feedback_to_screen "SNMP Trap Destinations: $SNMP_TRAP_DEST"
                        feedback_to_screen "$EXA_DB_IMAGE"
                    else
                        if [ "$DUAL_ASR_ENTRY" == "TRUE" ]; then
                            feedback_to_screen "ASR Manager IP:"

                        else
                            feedback_to_screen "ASR Manager IP:"
                            result=$(ASR_Return_5)
                            feedback_to_screen "$result"
                            feedback_to_screen ""
                            ASR_SNMP="ASR-8001-HI"
                            ASR_SNMP_DESC="ASR_Return_5"
                        fi
                    fi
                fi
            else
                asr_not_configured 
                ASR_SETUP_STATUS="Unconfigured"
            fi
        fi
        #End Bug fix Bug 20182652
    else
        feedback_to_screen "Skipping Exadata based checks"
        exit 1
    fi
    #END Node type validation
    #########################################

    #########################################
    # Validation Process
    #########################################
    #DB Only side Status process
    if [ $NODE_TYPE == "DB" ]; then
        feedback_to_screen ""
        feedback_to_screen "${PASS}#########ASR Validation#########${NO_COLOR}"
        #Check eth0 to make sure it is not on same subnet as the bond network
        if [ $OSTYPE == "SOLARIS" ]; then
            feedback_to_screen "Solaris DB Node IP Interface Check"
            solaris_ip_int_check
            feedback_to_screen ""
            feedback_to_screen "Solaris Perl Symbolic Link Location Validation"
            perl_location_validation
        else
            if [ $OSTYPE == "LINUX" ]; then
                feedback_to_screen "Linux DB Node IP Interface Check"
                linux_ip_int_check
            fi
        fi

        #BMC State check
        feedback_to_screen ""
        feedback_to_screen "Checking the BMC Status"
        bmc_validation
        #check for prefaulted Devices and notify console screen
        #check if PM function first before running hardware fault status
        if [ "$PMFUNCTION" == "FALSE" ]; then
            feedback_to_screen ""
            feedback_to_screen "Checking for Pre-Existing Errors"
            #Check for prefaulted devices
            #Disk Fault check
            feedback_to_screen ""
            feedback_to_screen "Checking for HDD Faults"
            disk_fault_Status
            #FMA ILOM Check
            feedback_to_screen ""
            feedback_to_screen "Checking for FMA Faults"
            if [ $BMC_STATUS -eq 0 ]; then
                ilom_fault_FMA_Status
            else
                screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                screen_output "Unable to Check for FMA Faults due to BMC Driver Issue"
            fi
            #ILOM Hardware Faults
            feedback_to_screen ""
            feedback_to_screen "Checking for ILOM Hardware Faults"
            if [ $BMC_STATUS -eq 0 ]; then
                ilom_fault_Status
            else
                screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                screen_output "Unable to Check for ILOM Hardware Faults due to BMC Driver Issue"
            fi
        fi
        #End PM check if set to false
        #########################################
        #DB Nodes Process Check
        if [ $NODE_TYPE == "DB" ]; then
            feedback_to_screen ""
            feedback_to_screen "DB Node validation Progress"
            feedback_to_screen ""
            #Bug 20182652
            if [ "$DBMCLI" == "TRUE" ]; then
                screen_output "Checking for ASR/Exadata Process Running"
                if [ $ASR_SETUP_STATUS == "Unconfigured" ]; then
                    feedback_to_screen ""
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "ASR Is not configured on Node"
                    screen_output "Please setup ASR per process from ASR Exadata Installation Guide"
                    screen_output "http://wd0338.oracle.com/archive/cd_ns/E13877_01/doc/doc.112/e13874.pdf"
                fi
            else
                #Check to see if exadata_mon_hw_asr.pl is running
                feedback_to_screen "Checking for ASR/Exadata Process Running"
                ps -ef | grep exadata_mon_hw_asr.pl | grep -v grep > /dev/null 2>&1
                if [ $? -ne 0 ]; then
                    feedback_to_screen ""
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "Exadata Monitoring Script is currently not running"
                    screen_output "Please Contact ASR Support for furhter Assistance"
                else
                    screen_output "Exadata Monitoring Script running as expected"
                fi
                feedback_to_screen ""
                feedback_to_screen "Validating Fromip"
                fromip_validation
                #Check to see if ASR is configured
                if [ $ASR_SETUP_STATUS == "Unconfigured" ]; then
                    feedback_to_screen ""
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "ASR Is not configured on Node"
                    screen_output "Please setup ASR per process from ASR Exadata Installation Guide"
                    screen_output "http://wd0338.oracle.com/archive/cd_ns/E13877_01/doc/doc.112/e13874.pdf"
                fi
            fi
            #End bug fix #Bug 20182652
        else
            #Cell Process
            if [ $NODE_TYPE == "CELL" ]; then
                feedback_to_screen "Cell Node validation Progress"
                screen_output "Checking for ASR/Exadata Process Running"
                if [ $ASR_SETUP_STATUS == "Unconfigured" ]; then
                    feedback_to_screen ""
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "ASR Is not configured on Node"
                    screen_output "Please setup ASR per process from ASR Exadata Installation Guide"
                    screen_output "http://wd0338.oracle.com/archive/cd_ns/E13877_01/doc/doc.112/e13874.pdf"
                fi
            fi
        fi
        #END DB Nodes Process
        #########################################
        if [ $TESTFUNCTION == "TRUE" ]; then
            if [ "$DBMCLI" == "TRUE" ]; then
                #DB Only Side Status Process
                dbos_validate_checks
                #Adding in the PM Functions
                if [ "$PMFUNCTION" == "TRUE" ]; then
                    feedback_to_screen ""
                    feedback_to_screen "${PASS}#########Proactive Hardware Check#########${NO_COLOR}"
                    pm_function
                fi
                #End of PMFunctions
            else
                feedback_to_screen ""
                feedback_to_screen "Validating ASR Configuration with Validate SNMP"
                if [ $BMC_STATUS -eq 0 ]; then
                    if [ $ASR_SETUP_STATUS == "Configured" ]; then
                        if [ $ASR_NET -eq "0" ]; then
                            ASR_TEST_VALUE="True"

                            #Send test event from DB Node
                            #Send test event from OS
                            feedback_to_screen ""
                            #Bug 20182652
                            if [ "$DBMCLI" == "FALSE" ]; then
                                feedback_to_screen "Exadata OS Validation"
                                screen_output "`$EXA_COMMAND -validate_snmp_subscriber -type asr`"
                                ASR_TEST_VALUE_OS="True"
                            fi
                            #End bug fix Bug 20182652
                            #Send test event from ILOM
                            feedback_to_screen ""
                            feedback_to_screen "Exadata ILOM Validation"
                            ilom_send_test
                        else
                            result=$(ASR_Return_7)
                            feedback_to_screen "$result"
                            ASR_TEST_VALUE="True"
                            echo "test $ASR_SNMP"
                            #Send test event from DB Node
                            #Send test event from OS
                            feedback_to_screen ""
                            #Bug 20182652
                            if [ "$DBMCLI" == "FALSE" ]; then
                                feedback_to_screen "Exadata OS Validation"
                                screen_output "`$EXA_COMMAND -validate_snmp_subscriber -type asr`"
                                ASR_TEST_VALUE_OS="True"
                            fi
                            #End bug fix Bug 20182652
                            #Send test event from ILOM
                            feedback_to_screen ""
                            feedback_to_screen "Exadata ILOM Validation"
                            ilom_send_test
                        fi        
                    else
                        result=$(ASR_Return_8)
                        feedback_to_screen "$result"
                        ASR_TEST_VALUE="False"
                        ASR_SNMP="ASR-8001-11"
                        ASR_SNMP_DESC="ASR_Return_8"
                    fi
                else
                    screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                    screen_output "Unable to Check SNMP for ILOM due to BMC Driver Issue"
                    ASR_TEST_VALUE="False"
                fi
            fi
        fi
        #Adding in the PM Functions
        if [ "$PMFUNCTION" == "TRUE" ]; then
            feedback_to_screen ""
            feedback_to_screen "${PASS}#########Proactive Hardware Check#########${NO_COLOR}"
            pm_function
        fi
        #End of PMFunctions
    else
        #Cell Only Side Status Process
        if [ $NODE_TYPE == "CELL" ]; then
            feedback_to_screen ""
            feedback_to_screen "${PASS}#########ASR Validation#########${NO_COLOR}"
            cell_validate_checks
            #Adding in the PM Functions
            if [ "$PMFUNCTION" == "TRUE" ]; then
                feedback_to_screen ""
                feedback_to_screen "${PASS}#########Proactive Hardware Check#########${NO_COLOR}"
                pm_function
            fi
            #End of PMFunctions
        fi
    fi
fi
#End Exadata Methods
#########################################
#Start BDA Methods
if [ $METHOD_TYPE == "BDA" ]; then
    #########################################
    #Start Node type validation
    #/opt/oracle/bda/install/state/mammoth-saved.params
    #SETUP_ASR_SERVER=false
    #ASR_HOST=
    #ASR_PORT=162
    #ASR_ENABLED=false
    ASR_ENB=`grep -i "ASR_ENABLED" $INSTALL_INFO | perl -pe 's/.*"(true|false)".*/\1/g'`
    if [ "$ASR_ENB" == "true" ]; then
        if [ -f $EXA_COMMAND ]; then
            if [ -f $EXADATA_MON/asrs.state ]; then
                if [ $? -eq 0 ]; then
                    ASR_SETUP_STATUS="Configured"
                    asrs_validate
                    ASR_MAN_IP_DB
                    if [ "$ASR_IP" != "N/A" ]; then
                        echo "ASR Manager IP: $ASR_IP"
                        SNMP_TRAP_DEST=`$EXA_COMMAND -get_snmp_subscribers -type asr | sed -e 's/^[ \t]*//'`
                        feedback_to_screen "SNMP Trap Destinations: $SNMP_TRAP_DEST"
                        feedback_to_screen "$EXA_DB_IMAGE"
                    else
                        feedback_to_screen "ASR Manager IP:"
                        result=$(ASR_Return_5)
                        feedback_to_screen "$result"
                        feedback_to_screen ""
                        ASR_SNMP="ASR-8001-HI"
                        ASR_SNMP_DESC="ASR_Return_5"
                        asr_not_configured 
                        ASR_SETUP_STATUS="Unconfigured"
                    fi
                fi
            else
                asr_not_configured 
                ASR_SETUP_STATUS="Unconfigured"
            fi
        fi
    else
       asr_not_configured 
       ASR_SETUP_STATUS="Unconfigured" 
    fi
    #END Node type validation
    #########################################
    #Start BDA Process Validation
    #Check to see if bda_mon_hw_asr.pl is running
    feedback_to_screen "Checking for ASR Process Running"
    ps -ef | grep bda_mon_hw_asr.pl | grep -v grep > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        feedback_to_screen ""
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "$METHOD_TYPE Monitoring Script is currently not running"
        screen_output "Please Contact ASR Support for furhter Assistance"
    else
        screen_output "$METHOD_TYPE Monitoring Script running as expected"
    fi
    #Check to see if ASR is configured
    if [ "$ASR_SETUP_STATUS" == "Unconfigured" ]; then
        feedback_to_screen ""
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "ASR Is not configured on Node"
        screen_output "Please setup ASR per process from ASR BDA Installation Guide"
        screen_output "http://www.oracle.com/technetwork/server-storage/engineered-systems/bigdata-appliance/documentation/index.html"
    fi
    #End BDA Process Validation
    #########################################
    # Validation Process
    #########################################
    if [ "$ASR_SETUP_STATUS" == "Configured" ]; then
        #BDA Status process
        feedback_to_screen "Linux IP Interface Check"
        linux_ip_int_check
        #fromip check
        feedback_to_screen ""
        feedback_to_screen "Validating Fromip"
        fromip_validation
        #BMC State check
        feedback_to_screen ""
        feedback_to_screen "Checking the BMC Status"
        bmc_validation
        #check for prefaulted Devices and notify console screen
        feedback_to_screen ""
        feedback_to_screen "Checking for Pre-Existing Errors"
        #Check for prefaulted devices
        #Disk Fault check
        feedback_to_screen ""
        feedback_to_screen "Checking for HDD Faults"
        disk_fault_Status
        #FMA ILOM Check
        feedback_to_screen ""
        feedback_to_screen "Checking for FMA Faults"
        if [ $BMC_STATUS -eq 0 ]; then
            ilom_fault_FMA_Status
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check for FMA Faults due to BMC Driver Issue"
        fi
        #ILOM Hardware Faults
        feedback_to_screen ""
        feedback_to_screen "Checking for ILOM Hardware Faults"
        if [ $BMC_STATUS -eq 0 ]; then
            ilom_fault_Status
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check for ILOM Hardware Faults due to BMC Driver Issue"
        fi
        if [ $TESTFUNCTION == "TRUE" ]; then
            feedback_to_screen ""
            feedback_to_screen "Validating ASR Configuration with Validate SNMP"
            if [ $BMC_STATUS -eq 0 ]; then
                if [ $ASR_SETUP_STATUS = "Configured" ]; then
                    if [ $ASR_NET -eq "0" ]; then
                        ASR_TEST_VALUE="True"
                        #Send test event from DB Node
                        #Send test event from OS
                        feedback_to_screen ""
                        feedback_to_screen "OS Validation"
                        screen_output `$EXA_COMMAND -validate_snmp_subscriber -type asr`
                        ASR_TEST_VALUE_OS="True"
                        #Send test event from ILOM
                        feedback_to_screen ""
                        feedback_to_screen "ILOM Validation"
                        ilom_send_test
                    else
                        result=$(ASR_Return_7)
                        feedback_to_screen "$result"
                        ASR_TEST_VALUE="True"
                        echo "test $ASR_SNMP"
                        #Send test event from DB Node
                        #Send test event from OS
                        feedback_to_screen ""
                        feedback_to_screen "OS Validation"
                        screen_output `$EXA_COMMAND -validate_snmp_subscriber -type asr`
                        ASR_TEST_VALUE_OS="True"
                        #Send test event from ILOM
                        feedback_to_screen ""
                        feedback_to_screen "ILOM Validation"
                        ilom_send_test
                    fi        
                else
                    result=$(ASR_Return_8)
                    feedback_to_screen "$result"
                    ASR_TEST_VALUE="False"
                    ASR_SNMP="ASR-8001-11"
                    ASR_SNMP_DESC="ASR_Return_8"
                fi
            else
                screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                screen_output "Unable to Check SNMP for ILOM due to BMC Driver Issue"
                ASR_TEST_VALUE="False"
            fi
        fi
    fi
fi
#########################################
#End BDA Methods
#########################################
#Start Exalytics
if [ $METHOD_TYPE == "Exalytics" ]; then
    #########################################
    #Start Node type validation
    if [ -f $EXA_COMMAND ]; then
        if [ -f $EXADATA_MON/asrs.state ]; then
            if [ $? -eq 0 ]; then
                ASR_SETUP_STATUS="Configured"
                asrs_validate
                ASR_MAN_IP_DB
                if [ "$ASR_IP" != "N/A" ]; then
                    feedback_to_screen "ASR Manager IP: $ASR_IP"
                    SNMP_TRAP_DEST=`$EXA_COMMAND -get_snmp_subscribers -type asr | sed -e 's/^[ \t]*//'`
                    feedback_to_screen "SNMP Trap Destinations: $SNMP_TRAP_DEST"
                    feedback_to_screen "$EXA_DB_IMAGE"
                else
                    feedback_to_screen "ASR Manager IP:"
                    result=$(ASR_Return_5)
                    feedback_to_screen "$result"
                    feedback_to_screen ""
                    ASR_SNMP="ASR-8001-HI"
                    ASR_SNMP_DESC="ASR_Return_5"
                fi
            fi
        else
            asr_not_configured 
            ASR_SETUP_STATUS="Unconfigured"
        fi
    fi

    #END Node type validation
    #########################################
    #Start Exalytics Process Validation
    #Check to see if bda_mon_hw_asr.pl is running
    echo "Checking for ASR Process Running"
    ps -ef | grep bda_mon_hw_asr.pl | grep -v grep > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        grep "bda_mon_hw_asr.pl" /etc/inittab > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            feedback_to_screen ""
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "$METHOD_TYPE Monitoring Script is currently not running"
            screen_output "Please Contact ASR Support for furhter Assistance"
        else
            feedback_to_screen "" >> /etc/inittab
            feedback_to_screen "#ASR Service" >> /etc/inittab
            feedback_to_screen "emh1:345:respawn:/opt/exalytics/asr/bda_mon_hw_asr.pl -server" >> /etc/inittab
            init q
        fi
    else
        screen_output "$METHOD_TYPE Monitoring Script running as expected"
    fi
    #Check to see if ASR is configured
    if [ $ASR_SETUP_STATUS == "Unconfigured" ]; then
        feedback_to_screen ""
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "ASR Is not configured on Node"
        screen_output "Please setup ASR per process from ASR Exalytics Installation Guide"
        screen_output "http://docs.oracle.com/cd/E27027_01/doc.1/e25369/asr.htm#sthref57"
    fi
    #End Exalytics Process Validation
    #########################################
    # Validation Process
    #########################################
    #BDA Status process
    feedback_to_screen "Linux IP Interface Check"
    linux_ip_int_check
    #fromip check
    feedback_to_screen ""
    feedback_to_screen "Validating Fromip"
    fromip_validation
    #BMC State check
    feedback_to_screen ""
    feedback_to_screen "Checking the BMC Status"
    bmc_validation
    #check for prefaulted Devices and notify console screen
    feedback_to_screen ""
    feedback_to_screen "Checking for Pre-Existing Errors"
    #Check for prefaulted devices
    #Disk Fault check
    feedback_to_screen ""
    feedback_to_screen "Checking for HDD Faults"
    disk_fault_Status
    #FMA ILOM Check
    feedback_to_screen ""
    feedback_to_screen "Checking for FMA Faults"
    if [ $BMC_STATUS -eq 0 ]; then
        ilom_fault_FMA_Status
    else
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "Unable to Check for FMA Faults due to BMC Driver Issue"
    fi
    #ILOM Hardware Faults
    feedback_to_screen ""
    feedback_to_screen "Checking for ILOM Hardware Faults"
    if [ $BMC_STATUS -eq 0 ]; then
        ilom_fault_Status
    else
        screen_output "${WARNING}#########Warning#########${NO_COLOR}"
        screen_output "Unable to Check for ILOM Hardware Faults due to BMC Driver Issue"
    fi
    if [ $TESTFUNCTION == "TRUE" ]; then
        feedback_to_screen ""
        feedback_to_screen "Validating ASR Configuration with Validate SNMP"
        if [ $BMC_STATUS -eq 0 ]; then
            if [ $ASR_SETUP_STATUS = "Configured" ]; then
                if [ $ASR_NET -eq "0" ]; then
                    ASR_TEST_VALUE="True"
                    #Send test event from DB Node
                    #Send test event from OS
                    feedback_to_screen ""
                    feedback_to_screen "OS Validation"
                    screen_output `$EXA_COMMAND -validate_snmp_subscriber -type asr`
                    ASR_TEST_VALUE_OS="True"
                    #Send test event from ILOM
                    feedback_to_screen ""
                    feedback_to_screen "ILOM Validation"
                    ilom_send_test
                else
                    result=$(ASR_Return_7)
                    feedback_to_screen "$result"
                    ASR_TEST_VALUE="True"
                    echo "test $ASR_SNMP"
                    #Send test event from DB Node
                    #Send test event from OS
                    feedback_to_screen ""
                    feedback_to_screen "OS Validation"
                    screen_output `$EXA_COMMAND -validate_snmp_subscriber -type asr`
                    ASR_TEST_VALUE_OS="True"
                    #Send test event from ILOM
                    feedback_to_screen ""
                    feedback_to_screen "ILOM Validation"
                    ilom_send_test
                fi        
            else
                result=$(ASR_Return_8)
                feedback_to_screen "$result"
                ASR_TEST_VALUE="False"
                ASR_SNMP="ASR-8001-11"
                ASR_SNMP_DESC="ASR_Return_8"
            fi
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check SNMP for ILOM due to BMC Driver Issue"
            ASR_TEST_VALUE="False"
        fi
    fi
fi
#########################################
#End Exalytics Methods
#########################################

#Start Exalogic Methods
if [ $METHOD_TYPE == "Exalogic" ]; then

    #Check for ASR Value has been entered in
    if [ -s $EXADATA_MON/asrs.state ]; then
        show_cached_asrs_state
        ASR_IP=`cat $EXADATA_MON/asrs.state | /usr/bin/awk -F: '{ print $1}'`
    else
        # 20562396 - need asrexachk to run standalone for pm_wrapper
        # add a 2nd cli arg (if num args > 1) to receive the IP/Port if Exalogic
        #
        # A. ip:port from pm_wrapper
        if [ "$#" -gt 1 ]; then
          screen_output "Running asrexachk with ASR_IP:ASR_PORT option" 
          # check for right format: IP:PORT (ip - colon - port)
          if [[ $2 =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}:[0-9]{1,3}$ ]]; then
              ASR_IP=`echo $2|sed 's/:.*//g'`
              ASR_PORT=`echo $2|sed 's/.*://g'`
              screen_output "(IP: $ASR_IP / Port: $ASR_PORT) is a valid ASR_IP:ASR_PORT combination"
          else
              # if wrong format, default to N/A which triggers B = ask for the info interactively
              screen_output "(IP: $ASR_IP / Port: $ASR_PORT) is not a valid ASR_IP:ASR_PORT combination, asking IP and Port in interactive mode" 
              ASR_IP="N/A"
          fi
        fi

        # B. ip/port not found or provided, ask for it
        if [ "$ASR_IP" == "N/A" ] || [ -z $ASR_IP ]; then
          #Create asrs.state file for future reference
          #Prompt for ASR Manager IP
          while true;
          do
              read -p "Enter ASR Manager IP: " exalogic_asr_ip
              if [[ $exalogic_asr_ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
                  ASR_IP="$exalogic_asr_ip"
                  break
              else
                  echo "Please enter a valid IP Address"
              fi
          done
          #Prompt for ASR Manager Port
          while true;
          do
              read -p "Enter ASR Manager SNMP Port: " exalogic_asr_port
              if [[ $exalogic_asr_port =~ ^[0-9]{1,3}$ ]]; then
                  ASR_PORT="$exalogic_asr_port"
                  break
              else
                  if [[ $exalogic_asr_port =~ ^[0-9]{1,4}$ ]]; then
                      ASR_PORT="$exalogic_asr_port"
                      break
                  else
                      echo "Please enter a valid SNMP Port"
                  fi
              fi
          done
        fi
    
        # write ip:port to asrs.state file
        echo $ASR_IP:$ASR_PORT > $EXADATA_MON/asrs.state
    fi

    #Start validation of Exalogic
    feedback_to_screen "$EXA_CELL_IMAGE"

    #BMC State check
    feedback_to_screen ""
    feedback_to_screen "Checking the BMC Status"
    bmc_validation
    if [ "$PMFUNCTION" == "FALSE" ]; then
      #check for prefaulted Devices and notify console screen
      feedback_to_screen ""
      feedback_to_screen "Checking for Pre-Existing Errors"
      #Check for prefaulted devices
      #Disk Fault check
      feedback_to_screen ""
      feedback_to_screen "Checking for HDD Faults"
      disk_fault_Status
      #FMA ILOM Check
      feedback_to_screen ""
      feedback_to_screen "Checking for FMA Faults"
      if [ $BMC_STATUS -eq 0 ]; then
          ilom_fault_FMA_Status
      else
          screen_output "${WARNING}#########Warning#########${NO_COLOR}"
          screen_output "Unable to Check for FMA Faults due to BMC Driver Issue"
      fi
      #ILOM Hardware Faults
      feedback_to_screen ""
      feedback_to_screen "Checking for ILOM Hardware Faults"
      if [ $BMC_STATUS -eq 0 ]; then
          ilom_fault_Status
      else
          screen_output "${WARNING}#########Warning#########${NO_COLOR}"
          screen_output "Unable to Check for ILOM Hardware Faults due to BMC Driver Issue"
      fi
      feedback_to_screen ""
    else
      feedback_to_screen ""
      feedback_to_screen "${PASS}#########Proactive Hardware Check#########${NO_COLOR}"
      pm_function
      # End of PMFunctions
    fi

    if [ $TESTFUNCTION == "TRUE" ]; then
        feedback_to_screen "Validating ASR Configuration with Validate SNMP"
        if [ $BMC_STATUS -eq 0 ]; then
            #Send test event from ILOM
            feedback_to_screen ""
            feedback_to_screen "ILOM Validation"
            ilom_send_test

        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check SNMP for ILOM due to BMC Driver Issue"
            ASR_TEST_VALUE="False"
        fi
    fi
fi
#########################################
#End Exalogic Methods
#########################################
#Start Super Cluster Methods
if [ $METHOD_TYPE == "SSC" ]; then
    #########################################
#Node type validation
    if [ -f /opt/oracle.cellos/ORACLE_CELL_NODE ]; then
        NODE_TYPE="CELL"
        #Cellcli retrieve trap destination of type ASR
        cellcli -e "list cell attributes snmpsubscriber" | grep -i "type=asr" > /dev/null 2>&1
        if [ $? -eq 0 ]; then
            cellcli -e list cell detail | grep -i notificationMethod | grep -i snmp > /dev/null 2>&1
            if [ $? -eq 0 ]; then
                ASR_SETUP_STATUS="Configured"
                
                ASR_MAN_IP_CELL
                feedback_to_screen "ASR Manager IP: $ASR_IP"
                CELL_LIST=`cellcli -e "list cell attributes snmpsubscriber"`
                feedback_to_screen "SNMP Trap Destinations: $CELL_LIST" | sed -e 's/^[ \t]*//'
                feedback_to_screen "Exadata Cell Image: $EXA_CELL_IMAGE"
            else
                cellcli -e list cell detail | grep -i notificationMethod | grep -i mail > /dev/null 2>&1
                if [ $? -eq 0 ]; then
                    feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                    feedback_to_screen "ASREXACHK has found a configuration issue with the Cell Notification Method."
                    feedback_to_screen "ASREXACHK returned value of 'mail' only"
                    feedback_to_screen "Running command to set value to include SNMP"
                    cellcli -e "alter cell notificationMethod='mail,snmp'" > /dev/null 2>&1
                    ASR_SETUP_STATUS="Configured"
                else
                    feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
                    feedback_to_screen "ASREXACHK has found a configuration issue with the Cell Notification Method."
                    feedback_to_screen "ASREXACHK returned value with no method set"
                    feedback_to_screen "Running command to set value to include SNMP"
                    cellcli -e "alter cell notificationMethod='snmp'" > /dev/null 2>&1
                    ASR_SETUP_STATUS="Configured"
                fi
            fi
        else
            asr_not_configured 
            ASR_SETUP_STATUS="Unconfigured"
            feedback_to_screen "${WARNING}#########Warning#########${NO_COLOR}"
            feedback_to_screen "Unable to determine configuration status of the cell node."
            feedback_to_screen "Please check to make sure the ASR Manager Hostname or IP are"
            feedback_to_screen "setup through the cellcli command to include the type of ASR"
            feedback_to_screen "Please reference the Oracle Exadata Administration guide for more"
            feedback_to_screen "details on cell commands for ASR as well"
            feedback_to_screen "as My Oracle Support Knowledge Article 1451026.1 for use with Grid Control"
        fi
    else
        #Check for ASR Value has been entered in
        if [ -s $EXADATA_MON/asrs.state ]; then
            show_cached_asrs_state
            ASR_IP=`cat $EXADATA_MON/asrs.state | /usr/bin/awk -F: '{ print $1}'`
        else
            #Create asrs.state file for future reference
            #Prompt for ASR Manager IP
            while true;
            do
                read -p "Enter ASR Manager IP: " exalogic_asr_ip
                if [[ $exalogic_asr_ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
                    ASR_IP="$exalogic_asr_ip"
                    break
                else
                    echo "Please enter a valid IP Address"
                fi
            done
            #Prompt for ASR Manager Port
            while true;
            do
                read -p "Enter ASR Manager SNMP Port: " exalogic_asr_port
                if [[ $exalogic_asr_port =~ ^[0-9]{1,3}$ ]]; then
                    ASR_PORT="$exalogic_asr_port"
                    break
                else
                    if [[ $exalogic_asr_port =~ ^[0-9]{1,4}$ ]]; then
                        ASR_PORT="$exalogic_asr_port"
                        break
                    else
                        echo "Please enter a valid SNMP Port"
                    fi
                fi
            done
            echo $ASR_IP:$ASR_PORT > $EXADATA_MON/asrs.state
        fi
    fi


#Start validation of SSC Cell
feedback_to_screen "$EXA_CELL_IMAGE"
#BMC State check
    feedback_to_screen ""
    feedback_to_screen "Checking the BMC Status"
    bmc_validation
    if [ "$PMFUNCTION" == "FALSE" ]; then
        #check for prefaulted Devices and notify console screen
        feedback_to_screen ""
        feedback_to_screen "Checking for Pre-Existing Errors"
        #Check for prefaulted devices
        #FMA ILOM Check
        feedback_to_screen ""
        feedback_to_screen "Checking for FMA Faults"
        if [ $BMC_STATUS -eq 0 ]; then
            ilom_fault_FMA_Status
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check for FMA Faults due to BMC Driver Issue"
        fi
        #ILOM Hardware Faults
        feedback_to_screen ""
        feedback_to_screen "Checking for ILOM Hardware Faults"
        if [ $BMC_STATUS -eq 0 ]; then
            ilom_fault_Status
        else
            screen_output "${WARNING}#########Warning#########${NO_COLOR}"
            screen_output "Unable to Check for ILOM Hardware Faults due to BMC Driver Issue"
        fi
        feedback_to_screen ""
        if [ $TESTFUNCTION == "TRUE" ]; then
            feedback_to_screen "Validating ASR Configuration with Validate SNMP"
            if [ $BMC_STATUS -eq 0 ]; then
                #Send test event from ILOM
                feedback_to_screen ""
                feedback_to_screen "ILOM Validation"
                ilom_send_test

            else
                screen_output "${WARNING}#########Warning#########${NO_COLOR}"
                screen_output "Unable to Check SNMP for ILOM due to BMC Driver Issue"
                ASR_TEST_VALUE="False"
            fi
        fi
        #Adding in the PM Functions
    else
        feedback_to_screen ""
        feedback_to_screen "${PASS}#########Proactive Hardware Check#########${NO_COLOR}"
        pm_function

        #End of PMFunctions
    
    fi
fi
#End Super Cluster Methods
#########################################
#Validation Success messages
if [ $TESTFUNCTION == "TRUE" ]; then
    if [ $METHOD_TYPE != "Exalogic" ]; then
        #OS validation Success Message
        if [ "$ASR_TEST_VALUE" == "True" ]; then
            if [ "$ASR_TEST_VALUE_OS" == "True" ]; then
                if [ "$ASR_TEST_VALUE_ILOM" == "True" ]; then
                    feedback_to_screen ""
                    feedback_to_screen ""
                    feedback_to_screen ""
                    feedback_to_screen "${PASS}Validation Complete${NO_COLOR}"
                    screen_output "Based on the Status, the email account registered to the ASR Manager"
                    screen_output "should receive 2 emails. One from the OS Hostname, and the"
                    screen_output "second from the ILOM Hostname"
                    screen_output "If 2 emails has not been received, please work with the ASR Backline"
                    screen_output "Team for further troubleshooting of the issue"
                    echo "Send ILOM Test Successful" >> $LOG
                    SNMP_CHECK="ASRExachk Validation Completed"
                else
                    echo "Send ILOM Test Failure" >> $LOG
                    SNMP_CHECK="ASRExachk ILOM Test Failure"
                fi
                echo "Send OS Test Successful" >> $LOG
            else
                echo "Send OS Test Failure" >> $LOG
                SNMP_CHECK="ASRExachk OS Test Failure"
            fi
        else
            echo "Configuration Failure" >> $LOG
            SNMP_CHECK="ASRexachk Configuration Failure"
        fi
    else
        if [ "$ASR_TEST_VALUE" == "True" ]; then
            if [ "$ASR_TEST_VALUE_ILOM" == "True" ]; then
                feedback_to_screen ""
                feedback_to_screen ""
                feedback_to_screen ""
                feedback_to_screen "${PASS}Validation Complete${NO_COLOR}"
                screen_output "Based on the Status, the email account registered to the ASR Manager"
                screen_output "should receive 2 emails. One from the OS Hostname, and the"
                screen_output "second from the ILOM Hostname"
                screen_output "If 2 emails has not been received, please work with the ASR Backline"
                screen_output "Team for further troubleshooting of the issue"
                echo "Send ILOM Test Successful" >> $LOG
                SNMP_CHECK="ASRExachk Validation Completed"
            else
                echo "Send ILOM Test Failure" >> $LOG
                SNMP_CHECK="ASRExachk ILOM Test Failure"
            fi
        else
            echo "Configuration Failure" >> $LOG
            SNMP_CHECK="ASRexachk Configuration Failure"
        fi
    fi
fi
#End of Validation Container Validation process

#bundling log files for PM process

if [ "$PMFUNCTION" == "TRUE" ]; then
    tar -cjf $LOG_DIR/asrexacheck_"$SERVER_TYPE"_"$SERIAL_NUM"_`date '+%Y-%m-%d_%H-%M'`.tar.bz2 -C $LOGDIR . > /dev/null 2>&1
    #put file location to screen
    echo ""
    echo "Proactive Support Bundle is asrexacheck_"$SERVER_TYPE"_"$SERIAL_NUM"_`date '+%Y-%m-%d_%H-%M'`.tar.bz2"
    echo "Located in $LOG_DIR"
    
fi
#end of log bundling
#send SNMPTrap
if [ "$TESTFUNCTION" == "TRUE" ]; then
    if [ $ASR_TEST_VALUE == "True" ]; then
        echo "$ASR_SNMP"
        if [ -z "$ASR_SNMP" ]; then
            ASR_SNMP="ASR-8000-WG"
            #echo "$ASR_SNMP"
            ASR_SNMP_DESC="Validation completed with no warnings"
            #echo "$ASR_SNMP_DESC"
        fi
        $SNMPTRAPCMD -v2c -c public $ASR_IP $UPTIME $sunHwTrapFaultDiagnosed \
        $sunHwTrapFaultMessageID         s "$ASR_SNMP" \
        $sunHwTrapSuspectComponentName   s "$ASR_SNMP_DESC" \
        $sunHwTrapProductManufacturer    s "Oracle Corporation" \
        $sunHwTrapProductName            s "$SNMP_SERVER" \
        $sunHwTrapProductSn              s "$SERIAL_NUM" \
        $sunHwTrapSystemIdentifier       s "$DEV_TYPE" \
        $sunHwTrapHostname               s "$HOSTNAME" \
        $sunHwTrapFirmwareVersion        s "$OSTYPE" \
        $sunHwTrapFirmwareBuild          s "$EXA_IMAGE" \
        $sunHwTrapFirmwareDate           s "`date`" \
        $sunHwTrapILOMVersion            s "$VERSION" \
        $sunHwTrapAdditionalInfo         s "$PM_CHECK" \
        $sunHwTrapAdditionalInfo         s "$SNMP_CHECK"
    fi
fi
#end SNMPTrap
# end debug
if $DEBUG; then
  set +x
  exec 2>&3 3>&-
fi

exit 0

