#!/usr/bin/perl # # IOCA - Isilon On-Cluster Analysis # Current version: 0.1224 # # Copyright (c) 2016-19 Dell Inc. or its subsidiaries. All Rights Reserved. # package IOCA; use warnings; use Cwd; use Term::ANSIColor; use Time::Local; ############################################################## # Global variables # Container for script and upgrade specific information my %RPS = (); @{$RPS{'Upgrade Plan'}} = (); $RPS{'Release Timestamp'} = 1548267898; # date +%s $RPS{'Version'} = "0.1224"; $RPS{'Issues'} = 0; $RPS{'DSP'} = "1.29"; $RPS{'DFP'} = "1.18"; $RPS{'IsiFw'} = "10.2.1"; $RPS{'CheckList'} = "healthcheck"; # Container for a description of each check our %CHECKS = (); # Container for check results, used for JSON output my @RESULTS = (); # Container for per node information my %NODE_INFO = (); # Container for cluster wide information my %CLUSTER_INFO = (); # Sorted list of lnns my @LNNS = (); # Conversion table used to lookup lnn from a devid my %LNN_FROM_DEVID = (); # Global variables used to store output during checks my @FAIL = (); my @CRITICAL = (); my @WARN = (); my @INFO = (); my @DETAILS = (); my @DEBUG = (); my %MISSING = (); # SUDO variable is used for compliance mode clusters my $SUDO = ""; sub main { # Initialize the script initRPSisiHealth(); # Run all checks in list &{$_}() for @{$CheckLists{$RPS{'CheckList'}}}; # If Issues is still 0, then no issues detected if ($RPS{'CheckList'} eq "pre-upgrade") { if ($RPS{'Issues'} == 0) { print "\nUPGRADE MAY PROCEED\n\n"; } else { print "\nUPGRADE ISSUE DETECTED\n\n"; } } # Develop the upgrade plan upgradePlan() if ($RPS{'UpgradePlan'}); # Print JSON print toJSON(\@RESULTS)."\n" if @RESULTS and $RPS{'JSON'}; } # Hash of Arrays of node firmware for each device BEGIN { our %CheckLists = ( "pre-upgrade" => [ "checkOneFSVersions", "checkPatches", "checkUpgradePath", "checkCapacity", "checkHealth", "checkEvents", "checkJobStatus", "checkPartitions", "checkServices", "checkProcesses", "checkIBInterfaces", "checkUptime", "checkUpgrade", "checkHardwareStatus", "checkBMCandCMC", "checkBootDisks", "checkMirrors", "checkMemory", "checkDrivesHealth", "checkDriveFirmware", "checkFCOF042415EE", "checkNodeFirmware", "checkNodeCompatibility", "checkSmartConnect", "checkGatewayPriority", "checkSyncIQ", "checkNDMP", "checkAggregationMode", "checkLACPSFP", "checkDiskpools", "checkAuthStatus", "checkNFS", "checkLicense", "checkISCSI", "checkAccessZones", "checkAspera", "checkContact", "checkEncoding", "checkFilepoolPolicies", "checkLegacyLDAP", "checkTimeZone", "checkKB527312", "checkKB519119", "checkKB507031", "checkKB521890", "checkKB469133", "checkKB524082", "checkKB516613", "checkKerberos8000", "checkLastZoneID8000", "checkSBR8000", "checkNFS8004", "checkSRS", "checkKB496582", "checkKB519388", "checkKB519890", "checkETAs", "checkBXENodes", ], "firmware" => [ "checkDriveFirmware", "checkNodeFirmware", ], "healthcheck" => [ "checkContact", "checkEmailSettings", "checkDriveFirmware", "checkETAs", "checkAggregationMode", "checkBMCandCMC", "checkBootDisks", "checkBXENodes", "checkDrivesHealth", "checkDriveLoad", "checkDriveStallTimeout", "checkGatewayPriority", "checkHardwareStatus", "checkIBInterfaces", "checkLACPSFP", "checkMemory", "checkMirrors", "checkNodeCompatibility", "checkNodes", "checkNodeFirmware", "checkAccessZones", "checkDiskpools", "checkFilepoolPolicies", "checkKerberos8000", "checkLastZoneID8000", "checkLegacyLDAP", "checkNFS8004", "checkOneFSVersions", "checkSBR8000", "checkKB507031", "checkKB519119", "checkKB521890", "checkKB469133", "checkKB516613", "checkFCOF042415EE", "checkUpgradePath", # "checkAspera", "checkAuthStatus", "checkCapacity", "checkEncoding", "checkSRS", "checkEvents", "checkFileSharing", "checkHDFS", "checkSPNs", "checkHealth", "checkIDI", "checkISCSI", "checkJobStatus", "checkJobHistory", "checkKB496582", "checkKB519890", "checkLicense", "checkLWIODLog", # "checkMessagesLog", # "checkNetstat", "checkListenQueue", "checkNFS", "checkOpenFiles", "checkPartitions", "checkPatches", "checkStoragePools", "checkProcesses", # "checkRoutingTables", "checkServices", "checkSmartConnect", "checkSnapshot", "checkSyncIQ", "checkTimeDrift", "checkTimeSync", "checkTimeZone", "checkUpgrade", "checkUptime", ], ); } BEGIN { our %DriveFirmware = ( "None" => undef, "1.18" => { "HDS725050KLA360" => "K2AOAD1A", "HGST HUS724020ALA640" => "MF6OAC50", "HGST HUS724030ALA640" => "MF8OAC50", "HGST HUS724040ALA640" => "MFAOAC50", "HGST HUS726060ALA640" => "AHIST1EZ", "HGST HUSMM1620ASS200" => "A204", "HGST HUSMM1640ASS200" => "A204", "HGST HUSMM1680ASS200" => "A204", "HGST HUSMM1680ASS205" => "D326", "HGST HUSMM8020ASS200" => "A210", "HGST HUSMM8040ASS200" => "A210", "HGST HUSMM8080ASS200" => "A210", "HITACHI HUC106030CSS600" => "A430", "HITACHI HUC106060CSS600" => "A430", "HITACHI HUC109030CSS600" => "A5B0", "HITACHI HUC109060CSS600" => "A5B0", "HITACHI HUC109090CSS600" => "A5B0", "HITACHI HUS156045VLS600" => "A760", "HITACHI HUSSL4010ASS600" => "A182", "HITACHI HUSSL4010BSS600" => "A1D0", "HITACHI HUSSL4020ASS600" => "A182", "HITACHI HUSSL4020BSS600" => "A1D0", "HITACHI HUSSL4040ASS600" => "A182", "HITACHI HUSSL4040BSS600" => "A1D0", "Hitachi HUA722010CLA330" => "JP4OA3PA", "Hitachi HUA722020ALA330" => "JKAOA3PA", "Hitachi HUA722050CLA330" => "JP2OA3PA", "Hitachi HUA723020ALA640" => "MK7OAA50", "Hitachi HUA723030ALA640" => "MKAOAA50", "SEAGATE ST3000NM0063" => "C007", "SEAGATE ST33000652SS" => "A006", "SEAGATE ST4000NM0063" => "C007", "ST1000NM0033-9ZM173" => "EN07", "ST2000NM0033-9ZM175" => "EN07", "ST3000NM0033-9ZM178" => "EN07", "ST4000NM0033-9ZM170" => "EN07", "STEC M8ISB2B-200UC" => "2832", "STEC MACH8 IOPS" => "2828", "STEC M8ISB2B-200UC" => "2832", "STEC MACH8 IOPS" => "2828", "WDC WD1002FBYS-02A6B0" => "03.02C06", }, ); } BEGIN { our %DriveSupport = ( "None" => undef, "1.29" => undef, "1.28" => { "HGST H5472606CLAR6000" => { "NDY0" => [ "N9L0" ], }, "HGST H5H72808CLAR8000" => { "MDY0" => [ "M9L0" ], }, "HGST HUC101890CSS205" => { "DDY0" => [ "DAA2" ], }, "HITACHI H5C10181CLAR1200" => { "DDY0" => [ "DAA2" ], }, "HITACHI H5C10186 CLAR600" => { "DDY0" => [ "DAA2" ], }, }, "1.26" => { "HGST HUSMM1680ASS205" => { "D32A" => [ "D252", "d322", "D326", "D328" ], }, "HITACHI H5472602CLAR2000" => { "DDW0" => [ "D930", "DD9H" ], }, "HITACHI H5472604CLAR4000" => { "DDW0" => [ "D930", "DD9H" ], }, "HITACHI HUSMM151CLAR1600" => { "D348" => [ "D344", "D346" ], }, "SEAGATE ST3000NM0063" => { "G007" => [ "0003", "0004", "0006", "C007", "F007" ], }, "SEAGATE ST4000NM0063" => { "G007" => [ "0003", "0004", "0006", "C007", "F007" ], }, "ST1000NM0033-9ZM173" => { "EN09" => [ "SN03", "SNG4", "EN06", "EN07" ], }, "ST2000NM0033-9ZM175" => { "EN09" => [ "SN03", "SNG4", "EN06", "EN07" ], }, "ST3000NM0033-9ZM178" => { "EN09" => [ "SN03", "SNG4", "EN06", "EN07" ], }, "ST4000NM0033-9ZM170" => { "EN09" => [ "SN02", "SN03", "SNG4", "EN06", "EN07" ], }, }, "1.24" => undef, "1.23" => { "HITACHI H5472602CLAR2000" => { "DD9H" => [ "D930" ], }, "HITACHI H5472604CLAR4000" => { "DD9H" => [ "D930" ], }, }, "1.22" => { "HGST HUSMM1680ASS205" => { "D328" => [ "D252", "d322", "D326" ], }, "HITACHI HUSMM151CLAR1600" => { "D346" => [ "D344" ], }, "HUS726020ALN610" => { "APGNTDCH" => [ "APGNT840" ], }, "HUS726040ALN610" => { "APGNTDCH" => [ "APGNT840" ], }, "SEAGATE ST3000NM0063" => { "F007" => [ "0003", "0004", "0006", "C007" ], }, "SEAGATE ST4000NM0063" => { "F007" => [ "0003", "0004", "0006", "C007" ], }, "ST8000NM0045-1RL112" => { "UG07" => [ "UG05" ], }, }, "1.21" => { "HGST HUC101812CSS200" => { "AD40" => [ "A9C1" ], }, "HGST HUC101830CSS200" => { "AD40" => [ "AAA2" ], }, "HGST HUC101860CSS200" => { "AD40" => [ "AAA2" ], }, "HGST HUC101890CSS200" => { "AD40" => [ "AAA2" ], }, "HUH728080ALN600" => { "A4EITD40" => [ "A4EIT921" ], }, "HUS726020ALA610" => { "A5EITD40" => [ "A5EIT922" ], }, "HUS726040ALA610" => { "A5EITD40" => [ "A5EIT922" ], }, "HUS726060ALN610" => { "APEITD40" => [ "APEIT922" ], }, "SEAGATE ST6000NM0114" => { "KFH5" => [ "KF05" ], }, }, "1.20" => undef, "1.19" => undef, "1.18" => { "SEAGATE ST3000NM0063" => { "C007" => [ "0003", "0004", "0006" ], }, "SEAGATE ST4000NM0063" => { "C007" => [ "0003", "0004", "0006" ], }, }, "1.17" => { "ST1000NM0033-9ZM173" => { "EN07" => [ "SN03", "SNG4", "EN06" ], }, "ST2000NM0033-9ZM175" => { "EN07" => [ "SN03", "SNG4", "EN06" ], }, "ST3000NM0033-9ZM178" => { "EN07" => [ "SN03", "SNG4", "EN06" ], }, "ST4000NM0033-9ZM170" => { "EN07" => [ "SN02", "SN03", "SNG4", "EN06" ], }, }, "1.16" => undef, "1.15" => { "HGST HUSMM1680ASS205" => { "D326" => [ "D252", "d322" ], }, }, "1.14" => { "HGST HUSMM1680ASS205" => { "D326" => [ "D252" ], }, }, "1.13" => undef, "1.12" => { "SEAGATE ST3000NM0063" => { "0006" => [ "0003", "0004" ], }, "SEAGATE ST4000NM0063" => { "0006" => [ "0003", "0004" ], }, "ST1000NM0033-9ZM173" => { "EN06" => [ "SN03", "SNG4" ], }, "ST2000NM0033-9ZM175" => { "EN06" => [ "SN03", "SNG4" ], }, "ST3000NM0033-9ZM178" => { "EN06" => [ "SN03", "SNG4" ], }, "ST4000NM0033-9ZM170" => { "EN06" => [ "SN02", "SN03", "SNG4" ], }, }, "1.11" => { "HGST HUSMM1680ASS205" => { "d322" => [ "D252" ], }, }, "1.10" => { "HDS725050KLA360" => { "K2AOAD1A" => [ "K2AOA10C", "K2AOA51A", "K2AOAB0A" ], }, "HGST HUS724020ALA640" => { "MF6OAC50" => [ "MF6OA8B0", "MF6OAAC0", "MF6OABW0" ], }, "HGST HUS724030ALA640" => { "MF8OAC50" => [ "MF8OA8B0", "MF8OAAC0", "MF8OABW0" ], }, "HGST HUS724040ALA640" => { "MFAOAC50" => [ "MFAOA8B0", "MFAOAAC0", "MFAOABW0" ], }, "HGST HUS726060ALA640" => { "AHIST1EZ" => [ "AHIST1ER" ], }, "HGST HUSMM1620ASS200" => { "A204" => [ "A100" ], }, "HGST HUSMM1640ASS200" => { "A204" => [ "A100" ], }, "HGST HUSMM1680ASS200" => { "A204" => [ "A100" ], }, "HGST HUSMM8020ASS200" => { "A210" => [ "A100", "A160" ], }, "HGST HUSMM8040ASS200" => { "A210" => [ "A100", "A160" ], }, "HGST HUSMM8080ASS200" => { "A210" => [ "A100", "A160" ], }, "HITACHI HUC106030CSS600" => { "A430" => [ "A180", "A202" ], }, "HITACHI HUC106060CSS600" => { "A430" => [ "A180", "A202" ], }, "HITACHI HUC109030CSS600" => { "A5B0" => [ "A222", "A2D0", "A350", "A440" ], }, "HITACHI HUC109060CSS600" => { "A5B0" => [ "A222", "A2D0", "A350", "A440" ], }, "HITACHI HUC109090CSS600" => { "A5B0" => [ "A222", "A2D0", "A350", "A440" ], }, "HITACHI HUS156045VLS600" => { "A760" => [ "A5D0" ], }, "HITACHI HUSSL4010ASS600" => { "A182" => [ "A170" ], "A18M" => [ "A131" ], }, "HITACHI HUSSL4010BSS600" => { "A1D0" => [ "A090", "A110", "A190" ], }, "HITACHI HUSSL4020ASS600" => { "A182" => [ "A170" ], "A18M" => [ "A131" ], }, "HITACHI HUSSL4020BSS600" => { "A1D0" => [ "A090", "A110", "A190" ], }, "HITACHI HUSSL4040ASS600" => { "A182" => [ "A170" ], "A18M" => [ "A131" ], }, "HITACHI HUSSL4040BSS600" => { "A1D0" => [ "A090", "A110", "A190" ], }, "Hitachi HUA722010CLA330" => { "JP4OA3PA" => [ "JP4OA3GB", "JP4OA3MA" ], }, "Hitachi HUA722020ALA330" => { "JKAOA3PA" => [ "JKAOA3GA", "JKAOA3MA", "JKAOA20N", "JKAOA28A" ], }, "Hitachi HUA722050CLA330" => { "JP2OA3PA" => [ "JP2OA3GB", "JP2OA3MA" ], }, "Hitachi HUA723020ALA640" => { "MK7OAA50" => [ "MK7OA6L0", "MK7OA8P0", "MK7OA580", "MK7OA5V0" ], }, "Hitachi HUA723030ALA640" => { "MKAOAA50" => [ "MKAOA6L0", "MKAOA8P0", "MKAOA3B0", "MKAOA580", "MKAOA5V0" ], }, "SEAGATE ST3000NM0063" => { "0004" => [ "0003" ], }, "SEAGATE ST33000652SS" => { "A006" => [ "0005", "0006" ], }, "SEAGATE ST4000NM0063" => { "0004" => [ "0003" ], }, "ST1000NM0033-9ZM173" => { "SNG4" => [ "SN03" ], }, "ST2000NM0033-9ZM175" => { "SNG4" => [ "SN03" ], }, "ST3000NM0033-9ZM178" => { "SNG4" => [ "SN03" ], }, "ST4000NM0033-9ZM170" => { "SNG4" => [ "SN02", "SN03" ], }, "STEC M8ISB2B-200UC" => { "2832" => [ "2686" ], }, "STEC MACH8 IOPS" => { "2828" => [ "2269", "2366", "2608", "2755" ], }, "WDC WD1002FBYS-02A6B0" => { "03.02C06" => [ "03.01C06" ], }, } ); } # Hash of Arrays of node firmware for each device BEGIN { our %NodeFirmware = ( "None" => undef, "10.2.1" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.07"], "CAR_Ermine" => [ "02.07"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.07"], "CMC_Ermine" => [ "02.07"], "CMC_HFHB" => [ "02.07"], "CMC_Yeti" => [ "02.07"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "DEadpt_moons" => [ "01.50"], "DEbcc_infinity" => [ "00.71"], "DEbios_banshee" => [ "33.18"], "DEbmc_bblock_moons" => [ "01.40"], "DEbmc_main" => [ "23.90"], "DEccdes" => [ "1.10.1+21.00"], "DEcdes" => [ "2.19.1+0.13.0+21.00"], "DEcmd_100d815pin" => [ "03.08.103.01"], "DEcmd_64pin" => [ "02.01.64.02"], "DEcmd_cat_64pin" => [ "02.05.102.02"], "DEcmd_mam_64pin" => [ "02.01.64.02"], "DEefi_sas_catamaran" => [ "0017.0000.0003.0000"], "DEeth_infinity" => [ "02.52"], "DEeth_sageville_infinity" => [ "0128.0000.0005.0195"], "DEfp_fpinffw" => [ "02.0c"], "DEfp_fpioc" => [ "00.0b"], "DEpost_banshee" => [ "28.05"], "DEps0_gen2_acbel_hv_zeus" => [ "04.12"], "DEps0_gen2_artesyn" => [ "02.14"], "DEps0_gen2_flex" => [ "02.07"], "DEps0_optimus_acbel" => [ "02.03"], "DEps0_optimus_flex" => [ "05.01"], "DEps0_ramjet_acbel" => [ "01.03"], "DEps1_gen2_acbel_hv_zeus" => [ "04.12"], "DEps1_gen2_artesyn" => [ "02.14"], "DEps1_gen2_flex" => [ "02.07"], "DEps1_optimus_acbel" => [ "02.03"], "DEps1_optimus_flex" => [ "05.01"], "DEps1_ramjet_acbel" => [ "01.03"], "DEsas_catamaran" => [ "0015.0002.0000.0000"], "DEsas_infinity" => [ "0001.0004.0000.11181"], "DEserdes_banshee" => [ "08.00"], "DEspime_banshee" => [ "02.01"], "DEssp_infinity" => [ "02.50"], "DEuefi_fw_volume_infinity" => [ "03.41"], "DEvrd0_banshee_stmicro" => [ "02.01"], "DEvrd1_banshee_stmicro" => [ "01.00"], "EPadpt_moons" => [ "01.50"], "EPbcc_infinity" => [ "00.71"], "EPbios_warthog" => [ "28.14"], "EPbmc_bblock_moons" => [ "01.40"], "EPbmc_main" => [ "23.90"], "EPccdes" => [ "1.10.1+21.00"], "EPcdes" => [ "2.19.1+0.13.0+21.00"], "EPcmd_100d815pin" => [ "03.08.103.01"], "EPcmd_64pin" => [ "02.01.64.02"], "EPcmd_cat_64pin" => [ "02.05.102.02"], "EPcmd_mam_64pin" => [ "02.01.64.02"], "EPefi_sas_catamaran" => [ "0017.0000.0003.0000"], "EPeth_infinity" => [ "02.52"], "EPfp_fpinffw" => [ "02.0c"], "EPfp_fpioc" => [ "00.0b"], "EPpost_warthog" => [ "22.05"], "EPps0_gen2_acbel_hv_zeus" => [ "04.12"], "EPps0_gen2_artesyn" => [ "02.14"], "EPps0_gen2_flex" => [ "02.07"], "EPps0_optimus_acbel" => [ "02.03"], "EPps0_optimus_flex" => [ "05.01"], "EPps0_ramjet_acbel" => [ "01.03"], "EPps1_gen2_acbel_hv_zeus" => [ "04.12"], "EPps1_gen2_artesyn" => [ "02.14"], "EPps1_gen2_flex" => [ "02.07"], "EPps1_optimus_acbel" => [ "02.03"], "EPps1_optimus_flex" => [ "05.01"], "EPps1_ramjet_acbel" => [ "01.03"], "EPsas_catamaran" => [ "0015.0002.0000.0000"], "EPsas_infinity" => [ "0001.0004.0000.11181"], "EPserdes_warthog" => [ "08.00"], "EPspime_warthog" => [ "07.00"], "EPssp_infinity" => [ "02.50"], "EPuefi_fw_volume_infinity" => [ "03.41"], "EPvrd0_warthog_stmicro" => [ "01.01"], "EPvrd1_warthog_stmicro" => [ "01.01"], "EPvrd2_warthog_stmicro" => [ "01.01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180d01","rp180d01+rp180d01"], "LOx2" => [ "rp180d01","rp180d01+rp180d01"], "LOx2-MLC-YD" => [ "rp180d01","rp180d01+rp180d01"], "Lsi" => [ "20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.40.5030+MT_1090110018","2.40.5030+ISL1090110018","4.8.930+MT_02F0140002","3.3.5"], "Mellanox-EN" => [ "2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN0" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN1" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox0" => [ "2.40.5030+EMC0000000004"], "Mellanox1" => [ "2.40.5030+EMC0000000004"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "XL710" => [ "5.04+80002505"], }, "10.1.6" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.07"], "CAR_Ermine" => [ "02.07"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.07"], "CMC_Ermine" => [ "02.07"], "CMC_HFHB" => [ "02.07"], "CMC_Yeti" => [ "02.07"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "DEadpt_moons" => [ "01.50"], "DEbcc_infinity" => [ "00.71"], "DEbios_banshee" => [ "33.18"], "DEbmc_bblock_moons" => [ "01.40"], "DEbmc_main" => [ "22.71"], "DEccdes" => [ "1.10.0+21.00"], "DEcdes" => [ "2.19.1+0.13.0+21.00"], "DEcmd_100d815pin" => [ "03.08.103.01"], "DEcmd_64pin" => [ "02.01.64.02"], "DEcmd_cat_64pin" => [ "02.05.102.02"], "DEcmd_mam_64pin" => [ "02.01.64.02"], "DEefi_sas_catamaran" => [ "0017.0000.0003.0000"], "DEeth_infinity" => [ "02.52"], "DEeth_sageville_infinity" => [ "0128.0000.0005.0195"], "DEfp_fpinffw" => [ "02.0c"], "DEfp_fpioc" => [ "00.0b"], "DEpost_banshee" => [ "26.86"], "DEps0_gen2_artesyn" => [ "02.14"], "DEps0_gen2_flex" => [ "02.07"], "DEps0_optimus_acbel" => [ "02.03"], "DEps0_optimus_flex" => [ "05.01"], "DEps0_ramjet_acbel" => [ "01.03"], "DEps1_gen2_artesyn" => [ "02.14"], "DEps1_gen2_flex" => [ "02.07"], "DEps1_optimus_acbel" => [ "02.03"], "DEps1_optimus_flex" => [ "05.01"], "DEps1_ramjet_acbel" => [ "01.03"], "DEsas_catamaran" => [ "0015.0002.0000.0000"], "DEsas_infinity" => [ "0001.0004.0000.11181"], "DEserdes_banshee" => [ "08.00"], "DEspime_banshee" => [ "02.01"], "DEssp_infinity" => [ "02.50"], "DEuefi_fw_volume_infinity" => [ "03.41"], "DEvrd0_banshee_stmicro" => [ "02.01"], "DEvrd1_banshee_stmicro" => [ "01.00"], "EPadpt_moons" => [ "01.50"], "EPbcc_infinity" => [ "00.71"], "EPbios_warthog" => [ "28.14"], "EPbmc_bblock_moons" => [ "01.40"], "EPbmc_main" => [ "22.71"], "EPccdes" => [ "1.10.0+21.00"], "EPcdes" => [ "2.19.1+0.13.0+21.00"], "EPcmd_100d815pin" => [ "03.08.103.01"], "EPcmd_64pin" => [ "02.01.64.02"], "EPcmd_cat_64pin" => [ "02.05.102.02"], "EPcmd_mam_64pin" => [ "02.01.64.02"], "EPefi_sas_catamaran" => [ "0017.0000.0003.0000"], "EPeth_infinity" => [ "02.52"], "EPfp_fpinffw" => [ "02.0c"], "EPfp_fpioc" => [ "00.0b"], "EPpost_warthog" => [ "20.86"], "EPps0_gen2_artesyn" => [ "02.14"], "EPps0_gen2_flex" => [ "02.07"], "EPps0_optimus_acbel" => [ "02.03"], "EPps0_optimus_flex" => [ "05.01"], "EPps0_ramjet_acbel" => [ "01.03"], "EPps1_gen2_artesyn" => [ "02.14"], "EPps1_gen2_flex" => [ "02.07"], "EPps1_optimus_acbel" => [ "02.03"], "EPps1_optimus_flex" => [ "05.01"], "EPps1_ramjet_acbel" => [ "01.03"], "EPsas_catamaran" => [ "0015.0002.0000.0000"], "EPsas_infinity" => [ "0001.0004.0000.11181"], "EPserdes_warthog" => [ "08.00"], "EPspime_warthog" => [ "07.00"], "EPssp_infinity" => [ "02.50"], "EPuefi_fw_volume_infinity" => [ "03.41"], "EPvrd0_warthog_stmicro" => [ "01.01"], "EPvrd1_warthog_stmicro" => [ "01.01"], "EPvrd2_warthog_stmicro" => [ "01.01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180d01","rp180d01+rp180d01"], "LOx2" => [ "rp180d01","rp180d01+rp180d01"], "LOx2-MLC-YD" => [ "rp180d01","rp180d01+rp180d01"], "Lsi" => [ "20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "3.3.5", "2.40.5030+MT_1090110018","2.40.5030+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN0" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN1" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox0" => [ "2.40.5030+EMC0000000004"], "Mellanox1" => [ "2.40.5030+EMC0000000004"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "XL710" => [ "5.04+80002505"], }, "10.1.3" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.07"], "CAR_Ermine" => [ "02.07"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.07"], "CMC_Ermine" => [ "02.07"], "CMC_HFHB" => [ "02.07"], "CMC_Yeti" => [ "02.07"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "DEadpt_moons" => [ "01.50"], "DEbcc_infinity" => [ "00.69"], "DEbios_banshee" => [ "33.15"], "DEbmc_bblock_moons" => [ "01.40"], "DEbmc_main" => [ "22.70"], "DEccdes" => [ "1.10.0+21.00"], "DEcdes" => [ "2.19.1+0.13.0+21.00"], "DEcmd_100d815pin" => [ "03.08.103.01"], "DEcmd_64pin" => [ "02.01.64.02"], "DEcmd_cat_64pin" => [ "02.05.102.02"], "DEcmd_mam_64pin" => [ "02.01.64.02"], "DEefi_sas_catamaran" => [ "0017.0000.0003.0000"], "DEeth_infinity" => [ "02.52"], "DEeth_sageville_infinity" => [ "0128.0000.0005.0195"], "DEfp_fpinffw" => [ "02.0b"], "DEfp_fpioc" => [ "00.0b"], "DEpost_banshee" => [ "26.84"], "DEps0_gen2_artesyn" => [ "02.03"], "DEps0_gen2_flex" => [ "02.07"], "DEps0_optimus_acbel" => [ "02.03"], "DEps0_optimus_flex" => [ "05.01"], "DEps0_ramjet_acbel" => [ "01.03"], "DEps1_gen2_artesyn" => [ "02.03"], "DEps1_gen2_flex" => [ "02.07"], "DEps1_optimus_acbel" => [ "02.03"], "DEps1_optimus_flex" => [ "05.01"], "DEps1_ramjet_acbel" => [ "01.03"], "DEsas_catamaran" => [ "0015.0002.0000.0000"], "DEsas_infinity" => [ "0001.0004.0000.11181"], "DEserdes_banshee" => [ "08.00"], "DEspime_banshee" => [ "02.01"], "DEssp_infinity" => [ "02.50"], "DEuefi_fw_volume_infinity" => [ "03.41"], "DEvrd0_banshee_stmicro" => [ "02.01"], "DEvrd1_banshee_stmicro" => [ "01.00"], "EPadpt_moons" => [ "01.50"], "EPbcc_infinity" => [ "00.69"], "EPbios_warthog" => [ "28.14"], "EPbmc_bblock_moons" => [ "01.40"], "EPbmc_main" => [ "22.70"], "EPccdes" => [ "1.10.0+21.00"], "EPcdes" => [ "2.19.1+0.13.0+21.00"], "EPcmd_100d815pin" => [ "03.08.103.01"], "EPcmd_64pin" => [ "02.01.64.02"], "EPcmd_cat_64pin" => [ "02.05.102.02"], "EPcmd_mam_64pin" => [ "02.01.64.02"], "EPefi_sas_catamaran" => [ "0017.0000.0003.0000"], "EPeth_infinity" => [ "02.52"], "EPfp_fpinffw" => [ "02.0b"], "EPfp_fpioc" => [ "00.0b"], "EPpost_warthog" => [ "20.84"], "EPps0_gen2_artesyn" => [ "02.03"], "EPps0_gen2_flex" => [ "02.07"], "EPps0_optimus_acbel" => [ "02.03"], "EPps0_optimus_flex" => [ "05.01"], "EPps0_ramjet_acbel" => [ "01.03"], "EPps1_gen2_artesyn" => [ "02.03"], "EPps1_gen2_flex" => [ "02.07"], "EPps1_optimus_acbel" => [ "02.03"], "EPps1_optimus_flex" => [ "05.01"], "EPps1_ramjet_acbel" => [ "01.03"], "EPsas_catamaran" => [ "0015.0002.0000.0000"], "EPsas_infinity" => [ "0001.0004.0000.11181"], "EPserdes_warthog" => [ "08.00"], "EPspime_warthog" => [ "07.00"], "EPssp_infinity" => [ "02.50"], "EPuefi_fw_volume_infinity" => [ "03.41"], "EPvrd0_warthog_stmicro" => [ "01.01"], "EPvrd1_warthog_stmicro" => [ "01.01"], "EPvrd2_warthog_stmicro" => [ "01.01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180d01","rp180d01+rp180d01"], "LOx2" => [ "rp180d01","rp180d01+rp180d01"], "LOx2-MLC-YD" => [ "rp180d01","rp180d01+rp180d01"], "Lsi" => [ "20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.40.5030+MT_1090110018","2.40.5030+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN0" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN1" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox0" => [ "2.40.5030+EMC0000000004"], "Mellanox1" => [ "2.40.5030+EMC0000000004"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "XL710" => [ "5.04+80002505"], }, "10.1.2" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.07"], "CAR_Ermine" => [ "02.07"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.07"], "CMC_Ermine" => [ "02.07"], "CMC_HFHB" => [ "02.07"], "CMC_Yeti" => [ "02.07"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "DEadpt_moons" => [ "01.50"], "DEbcc_infinity" => [ "00.69"], "DEbios_banshee" => [ "33.15"], "DEbmc_bblock_moons" => [ "01.40"], "DEbmc_main" => [ "22.70"], "DEccdes" => [ "1.10.0+21.00"], "DEcdes" => [ "2.19.1+0.13.0+21.00"], "DEcmd_100d815pin" => [ "03.08.103.01"], "DEcmd_64pin" => [ "02.01.64.02"], "DEcmd_cat_64pin" => [ "02.05.102.02"], "DEcmd_mam_64pin" => [ "02.01.64.02"], "DEefi_sas_catamaran" => [ "0017.0000.0003.0000"], "DEeth_infinity" => [ "02.52"], "DEeth_sageville_infinity" => [ "0128.0000.0005.0195"], "DEfp_fpinffw" => [ "02.0b"], "DEfp_fpioc" => [ "00.0b"], "DEpost_banshee" => [ "26.84"], "DEps0_gen2_artesyn" => [ "02.03"], "DEps0_gen2_flex" => [ "02.07"], "DEps0_optimus_acbel" => [ "02.03"], "DEps0_optimus_flex" => [ "05.01"], "DEps0_ramjet_acbel" => [ "01.03"], "DEps1_gen2_artesyn" => [ "02.03"], "DEps1_gen2_flex" => [ "02.07"], "DEps1_optimus_acbel" => [ "02.03"], "DEps1_optimus_flex" => [ "05.01"], "DEps1_ramjet_acbel" => [ "01.03"], "DEsas_catamaran" => [ "0015.0002.0000.0000"], "DEsas_infinity" => [ "0001.0004.0000.11181"], "DEserdes_banshee" => [ "08.00"], "DEspime_banshee" => [ "02.01"], "DEssp_infinity" => [ "02.50"], "DEuefi_fw_volume_infinity" => [ "03.41"], "DEvrd0_banshee_stmicro" => [ "02.01"], "DEvrd1_banshee_stmicro" => [ "01.00"], "EPadpt_moons" => [ "01.50"], "EPbcc_infinity" => [ "00.69"], "EPbios_warthog" => [ "28.14"], "EPbmc_bblock_moons" => [ "01.40"], "EPbmc_main" => [ "22.70"], "EPccdes" => [ "1.10.0+21.00"], "EPcdes" => [ "2.19.1+0.13.0+21.00"], "EPcmd_100d815pin" => [ "03.08.103.01"], "EPcmd_64pin" => [ "02.01.64.02"], "EPcmd_cat_64pin" => [ "02.05.102.02"], "EPcmd_mam_64pin" => [ "02.01.64.02"], "EPefi_sas_catamaran" => [ "0017.0000.0003.0000"], "EPeth_infinity" => [ "02.52"], "EPfp_fpinffw" => [ "02.0b"], "EPfp_fpioc" => [ "00.0b"], "EPpost_warthog" => [ "20.84"], "EPps0_gen2_artesyn" => [ "02.03"], "EPps0_gen2_flex" => [ "02.07"], "EPps0_optimus_acbel" => [ "02.03"], "EPps0_optimus_flex" => [ "05.01"], "EPps0_ramjet_acbel" => [ "01.03"], "EPps1_gen2_artesyn" => [ "02.03"], "EPps1_gen2_flex" => [ "02.07"], "EPps1_optimus_acbel" => [ "02.03"], "EPps1_optimus_flex" => [ "05.01"], "EPps1_ramjet_acbel" => [ "01.03"], "EPsas_catamaran" => [ "0015.0002.0000.0000"], "EPsas_infinity" => [ "0001.0004.0000.11181"], "EPserdes_warthog" => [ "08.00"], "EPspime_warthog" => [ "07.00"], "EPssp_infinity" => [ "02.50"], "EPuefi_fw_volume_infinity" => [ "03.41"], "EPvrd0_warthog_stmicro" => [ "01.01"], "EPvrd1_warthog_stmicro" => [ "01.01"], "EPvrd2_warthog_stmicro" => [ "01.01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180c01","rp180c01+rp180c01"], "LOx2" => [ "rp180c01","rp180c01+rp180c01"], "LOx2-MLC-YD" => [ "rp180c01","rp180c01+rp180c01"], "Lsi" => [ "20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.40.5030+MT_1090110018","2.40.5030+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN0" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN1" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox0" => [ "2.40.5030+EMC0000000004"], "Mellanox1" => [ "2.40.5030+EMC0000000004"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "XL710" => [ "5.04+80002505"], }, "10.1.1" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.07"], "CAR_Ermine" => [ "02.07"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.07"], "CMC_Ermine" => [ "02.07"], "CMC_HFHB" => [ "02.07"], "CMC_Yeti" => [ "02.07"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "DEadpt_moons" => [ "01.50"], "DEbcc_infinity" => [ "00.69"], "DEbios_banshee" => [ "33.15"], "DEbmc_bblock_moons" => [ "01.40"], "DEbmc_main" => [ "21.91"], "DEccdes" => [ "1.10.0+21.00"], "DEcdes" => [ "2.19.1+0.13.0+21.00"], "DEcmd_100d815pin" => [ "03.08.103.01"], "DEcmd_64pin" => [ "02.01.64.02"], "DEcmd_cat_64pin" => [ "02.05.102.02"], "DEcmd_mam_64pin" => [ "02.01.64.02"], "DEefi_sas_catamaran" => [ "0017.0000.0003.0000"], "DEeth_infinity" => [ "02.52"], "DEeth_sageville_infinity" => [ "0128.0000.0005.0195"], "DEfp_fpinffw" => [ "02.0b"], "DEfp_fpioc" => [ "00.08"], "DEpost_banshee" => [ "26.81"], "DEps0_gen2_artesyn" => [ "02.02"], "DEps0_gen2_flex" => [ "02.07"], "DEps0_optimus_acbel" => [ "02.03"], "DEps0_optimus_flex" => [ "05.01"], "DEps0_ramjet_acbel" => [ "01.03"], "DEps1_gen2_artesyn" => [ "02.02"], "DEps1_gen2_flex" => [ "02.07"], "DEps1_optimus_acbel" => [ "02.03"], "DEps1_optimus_flex" => [ "05.01"], "DEps1_ramjet_acbel" => [ "01.03"], "DEsas_catamaran" => [ "0015.0002.0000.0000"], "DEsas_infinity" => [ "0001.0004.0000.11181"], "DEserdes_banshee" => [ "08.00"], "DEspime_banshee" => [ "02.01"], "DEssp_infinity" => [ "01.90"], "DEuefi_fw_volume_infinity" => [ "03.41"], "DEvrd0_banshee_stmicro" => [ "02.01"], "DEvrd1_banshee_stmicro" => [ "01.00"], "EPadpt_moons" => [ "01.50"], "EPbcc_infinity" => [ "00.69"], "EPbios_warthog" => [ "28.14"], "EPbmc_bblock_moons" => [ "01.40"], "EPbmc_main" => [ "21.91"], "EPccdes" => [ "1.10.0+21.00"], "EPcdes" => [ "2.19.1+0.13.0+21.00"], "EPcmd_100d815pin" => [ "03.08.103.01"], "EPcmd_64pin" => [ "02.01.64.02"], "EPcmd_cat_64pin" => [ "02.05.102.02"], "EPcmd_mam_64pin" => [ "02.01.64.02"], "EPefi_sas_catamaran" => [ "0017.0000.0003.0000"], "EPeth_infinity" => [ "02.52"], "EPfp_fpinffw" => [ "02.0b"], "EPfp_fpioc" => [ "00.08"], "EPpost_warthog" => [ "20.81"], "EPps0_gen2_artesyn" => [ "02.02"], "EPps0_gen2_flex" => [ "02.07"], "EPps0_optimus_acbel" => [ "02.03"], "EPps0_optimus_flex" => [ "05.01"], "EPps0_ramjet_acbel" => [ "01.03"], "EPps1_gen2_artesyn" => [ "02.02"], "EPps1_gen2_flex" => [ "02.07"], "EPps1_optimus_acbel" => [ "02.03"], "EPps1_optimus_flex" => [ "05.01"], "EPps1_ramjet_acbel" => [ "01.03"], "EPsas_catamaran" => [ "0015.0002.0000.0000"], "EPsas_infinity" => [ "0001.0004.0000.11181"], "EPserdes_warthog" => [ "08.00"], "EPspime_warthog" => [ "07.00"], "EPssp_infinity" => [ "01.90"], "EPuefi_fw_volume_infinity" => [ "03.41"], "EPvrd0_warthog_stmicro" => [ "01.01"], "EPvrd1_warthog_stmicro" => [ "01.01"], "EPvrd2_warthog_stmicro" => [ "01.01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180c01","rp180c01+rp180c01"], "LOx2" => [ "rp180c01","rp180c01+rp180c01"], "LOx2-MLC-YD" => [ "rp180c01","rp180c01+rp180c01"], "Lsi" => [ "20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.40.5030+MT_1090110018","2.40.5030+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN0" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN1" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox0" => [ "2.40.5030+EMC0000000004"], "Mellanox1" => [ "2.40.5030+EMC0000000004"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "XL710" => [ "5.04+80002505"], }, "10.0.1" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.05"], "CAR_Ermine" => [ "02.05"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.05"], "CMC_Ermine" => [ "02.05"], "CMC_HFHB" => [ "02.05"], "CMC_Yeti" => [ "02.05"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "DEadpt_moons" => [ "01.50"], "DEbcc_infinity" => [ "00.64"], "DEbios_banshee" => [ "29.12"], "DEbmc_bblock_moons" => [ "01.40"], "DEbmc_main" => [ "21.01"], "DEcdes" => [ "2.19.1+0.13.0+21.00"], "DEcmd_100d815pin" => [ "03.07.101.01"], "DEcmd_64pin" => [ "02.01.64.02"], "DEeth_infinity" => [ "02.52"], "DEeth_sageville_infinity" => [ "0128.0000.0005.0195"], "DEfp_fpinffw" => [ "02.0a"], "DEfp_fpioc" => [ "00.08"], "DEpost_banshee" => [ "24.70"], "DEps0_gen2_artesyn" => [ "02.02"], "DEps0_gen2_flex" => [ "02.07"], "DEps0_optimus_acbel" => [ "02.03"], "DEps0_optimus_flex" => [ "05.01"], "DEps0_ramjet_acbel" => [ "01.03"], "DEps1_gen2_artesyn" => [ "02.02"], "DEps1_gen2_flex" => [ "02.07"], "DEps1_optimus_acbel" => [ "02.03"], "DEps1_optimus_flex" => [ "05.01"], "DEps1_ramjet_acbel" => [ "01.03"], "DEsas_infinity" => [ "0001.0004.0000.11146"], "DEserdes_banshee" => [ "06.00"], "DEspime_banshee" => [ "02.01"], "DEssp_infinity" => [ "01.55"], "DEuefi_fw_volume_infinity" => [ "01.36"], "DEvrd0_banshee_stmicro" => [ "02.01"], "DEvrd1_banshee_stmicro" => [ "01.00"], "EPadpt_moons" => [ "01.50"], "EPbcc_infinity" => [ "00.64"], "EPbios_warthog" => [ "24.17"], "EPbmc_bblock_moons" => [ "01.40"], "EPbmc_main" => [ "21.01"], "EPcdes" => [ "2.19.1+0.13.0+21.00"], "EPcmd_100d815pin" => [ "03.07.101.01"], "EPcmd_64pin" => [ "02.01.64.02"], "EPeth_infinity" => [ "02.52"], "EPfp_fpinffw" => [ "02.0a"], "EPfp_fpioc" => [ "00.08"], "EPpost_warthog" => [ "18.70"], "EPps0_gen2_artesyn" => [ "02.02"], "EPps0_gen2_flex" => [ "02.07"], "EPps0_optimus_acbel" => [ "02.03"], "EPps0_optimus_flex" => [ "05.01"], "EPps0_ramjet_acbel" => [ "01.03"], "EPps1_gen2_artesyn" => [ "02.02"], "EPps1_gen2_flex" => [ "02.07"], "EPps1_optimus_acbel" => [ "02.03"], "EPps1_optimus_flex" => [ "05.01"], "EPps1_ramjet_acbel" => [ "01.03"], "EPsas_infinity" => [ "0001.0004.0000.11146"], "EPserdes_warthog" => [ "05.00"], "EPspime_warthog" => [ "07.00"], "EPssp_infinity" => [ "01.55"], "EPuefi_fw_volume_infinity" => [ "01.36"], "EPvrd0_warthog_stmicro" => [ "01.01"], "EPvrd1_warthog_stmicro" => [ "01.01"], "EPvrd2_warthog_stmicro" => [ "01.01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180c01","rp180c01+rp180c01"], "LOx2" => [ "rp180c01","rp180c01+rp180c01"], "LOx2-MLC-YD" => [ "rp180c01","rp180c01+rp180c01"], "Lsi" => [ "20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.40.5030+MT_1090110018","2.40.5030+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN0" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox-EN1" => [ "2.40.5030+EMC0000000002","2.40.5030+MT_1090111023","2.40.5030+EMC1090111023"], "Mellanox0" => [ "2.40.5030+EMC0000000004"], "Mellanox1" => [ "2.40.5030+EMC0000000004"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "XL710" => [ "5.04+80002505"], }, "9.3.5" => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "02.05"], "CAR_Ermine" => [ "02.05"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "02.05"], "CMC_Ermine" => [ "02.05"], "CMC_HFHB" => [ "02.05"], "CMC_Yeti" => [ "02.05"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180c01","rp180c01+rp180c01"], "LOx2" => [ "rp180c01","rp180c01+rp180c01"], "LOx2-MLC-YD" => [ "rp180c01","rp180c01+rp180c01"], "Lsi" => [ "6.28.00.00+01.28.02.00+1.17","1.023","6.28.00.00+01.28.02.00+1.00","0.99c","20.00.04.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.30.8020+MT_1090110018","2.30.8020+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.34.5000+MT_1090111023"], "PMC8056" => [ "2.13.0+0.11.0+21.00"], "QLogic-NX2" => [ "7.13.0"], "Vitesse0" => [ "0.99c","1.024"], "Vitesse1" => [ "1.024"], }, '9.3.4' => { "BMCSDR_A100" => [ "01.07"], "CAR_Colossus" => [ "01.04"], "CAR_Ermine" => [ "00.0a"], "CFFPS1_Blastoff" => [ "03.03"], "CFFPS1_Octane" => [ "09.05"], "CFFPS1_Octane_Acbel" => [ "0a.01"], "CFFPS1_Optimus" => [ "04.14","02.02"], "CFFPS1_Optimus_Acbel" => [ "02.02"], "CFFPS1_Optimus_Flex" => [ "04.14"], "CFFPS1_Ramjet" => [ "06.06"], "CFFPS1_Ramjet_Acbel" => [ "01.03"], "CFFPS2_Blastoff" => [ "03.03"], "CFFPS2_Octane" => [ "09.05"], "CFFPS2_Octane_Acbel" => [ "0a.01"], "CFFPS2_Optimus" => [ "04.14","02.02"], "CFFPS2_Optimus_Acbel" => [ "02.02"], "CFFPS2_Optimus_Flex" => [ "04.14"], "CFFPS2_Ramjet" => [ "06.06"], "CFFPS2_Ramjet_Acbel" => [ "01.03"], "CMCSDR_Colossus" => [ "00.08"], "CMCSDR_Ermine" => [ "00.03"], "CMCSDR_Honeybadger" => [ "00.0B"], "CMCSDR_Hotfoot" => [ "00.0B"], "CMCSDR_Yeti" => [ "00.06"], "CMC_Colossus" => [ "01.04"], "CMC_Ermine" => [ "00.0a"], "CMC_HFHB" => [ "01.02"], "CMC_Yeti" => [ "00.0b"], "CMD_Colossus" => [ "02.0b+49+01"], "CMD_Ermine" => [ "02.0d+4c+01"], "IsilonFPV1" => [ "UI.01.36"], "IsilonIB" => [ "4.8.930+205-0002-05_A"], "LOx" => [ "rp180c01","rp180c01+rp180c01"], "LOx2" => [ "rp180c01","rp180c01+rp180c01"], "LOx2-MLC-YD" => [ "rp180c01","rp180c01+rp180c01"], "Lsi" => [ "6.28.00.00+01.28.02.00+1.17","1.023","6.28.00.00+01.28.02.00+1.00","0.99c","16.00.01.00","17.00.01.00","13.00.66.00"], "LsiExp0" => [ "0910+0210"], "LsiExp1" => [ "0910+0210"], "Mellanox" => [ "2.30.8020+MT_1090110018","2.30.8020+ISL1090110018","4.8.930+MT_02F0140002"], "Mellanox-EN" => [ "2.34.5000+MT_1090111023"], "PMC8056" => [ "0.11.1+0.7.0+00.15-transfr"], "QLogic-NX2" => [ "7.13.0"], "Vitesse0" => [ "0.99c","1.024"], "Vitesse1" => [ "1.024"], }, ); } # Supported out-of-family upgrade paths # Source => Target versions # Highest Sources at the top # Target versions are the lowest supported version that can be upgraded to at each family excluding within family # Target versions MUST be 4 decimal separated digits in length # Target versions are generated based on the keys when no target is found in the list # The only EOSL code that should be upgraded to is 6.5.5.29 as an intermediary code to get to 7.1.1+ BEGIN { our %Code_Matrix = ( "8.1.3.0" => [ ], "8.1.2.0" => [ "8.1.3.0" ], "8.1.1.1" => [ "8.1.3.0", "8.1.2.0" ], "8.1.1.0" => [ "8.1.3.0", "8.1.2.0" ], "8.1.0.4" => [ "8.1.3.0", "8.1.2.0" ], "8.1.0.3" => [ "8.1.3.0", "8.1.2.0" ], "8.1.0.2" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0" ], "8.1.0.1" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0" ], "8.1.0.0" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0" ], "8.0.1.2" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.2" ], "8.0.1.1" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.1" ], "8.0.1.0" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0" ], "8.0.0.7" => [ "8.1.3.0", "8.1.2.0" ], "8.0.0.6" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.2" ], "8.0.0.5" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.1", "8.0.1.2" ], "8.0.0.4" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.1" ], "8.0.0.3" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.1" ], "8.0.0.2" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.1" ], "8.0.0.1" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0" ], "8.0.0.0" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0" ], "7.2.1.6" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.1", "8.0.1.2", "8.0.0.5" ], "7.2.1.5" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.1", "8.0.0.4" ], "7.2.1.4" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.1", "8.0.0.3" ], "7.2.1.3" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.2" ], "7.2.1.2" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.1" ], "7.2.1.1" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.0" ], "7.2.1.0" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.0" ], "7.2.0.5" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.1", "7.2.1.2" ], "7.2.0.4" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.0", "7.2.1.1" ], "7.2.0.3" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.0", "7.2.1.1" ], "7.2.0.2" => [ "8.1.3.0", "8.1.2.0", "8.1.1.0", "8.1.0.0", "8.0.1.0", "8.0.0.0", "7.2.1.0" ], "7.2.0.1" => [ ], "7.2.0.0" => [ ], "7.1.1.11"=> [ "8.1.2.0", "8.1.1.0", "8.0.1.1", "8.0.0.4", "7.2.1.5" ], "7.1.1.9" => [ "8.0.1.0", "8.0.0.1", "7.2.1.3" ], "7.1.1.8" => [ "8.0.1.0", "8.0.0.1", "7.2.1.2" ], "7.1.1.7" => [ "8.0.1.0", "8.0.0.0", "7.2.1.1", "7.2.0.4" ], "7.1.1.6" => [ "8.0.1.0", "8.0.0.0", "7.2.1.1", "7.2.0.3" ], "7.1.1.5" => [ "8.0.1.0", "8.0.0.0", "7.2.1.1", "7.2.0.3" ], "7.1.1.4" => [ "8.0.1.0", "8.0.0.0", "7.2.1.0", "7.2.0.2" ], "7.1.1.3" => [ "7.2.0.1" ], "7.1.1.2" => [ "7.2.0.1" ], "7.1.1.1" => [ "7.2.0.0" ], "7.1.1.0" => [ "7.2.0.0" ], "7.1.0" => [ "7.2.0.1", "7.1.1.2" ], "7.0" => [ "7.2.0.2", "7.1.1.3" ], "6.5.5.29"=> [ "7.1.1.1", "7.1.0.0" ], "6.5" => [ "7.1.1.1", "7.1.0.0" ], "6.0" => [ "6.5.5.29" ], ); } # Activity names BEGIN { our %Activities = ( 'Drive Firmware' => { 5 => "Drive Firmware Upgrade 1-5 nodes", 10 => "Drive Firmware Upgrade 6-10 nodes", 15 => "Drive Firmware Upgrade 11-15 nodes", 20 => "Drive Firmware Upgrade 16-20 nodes", 25 => "Drive Firmware Upgrade 21-25 nodes", 30 => "Drive Firmware Upgrade 26-30 nodes", 35 => "Drive Firmware Upgrade 31-35 nodes", 40 => "Drive Firmware Upgrade 36-40 nodes", # 45 => "Drive Firmware Upgrade 41-45 nodes", # 50 => "Drive Firmware Upgrade 46-50 nodes", # 55 => "Drive Firmware Upgrade 51-55 nodes", # 60 => "Drive Firmware Upgrade 56-60 nodes", }, 'Node Firmware' => { 5 => "Node Firmware Upgrade 1-5 nodes", 10 => "Node Firmware Upgrade 6-10 nodes", 15 => "Node Firmware Upgrade 11-15 nodes", 20 => "Node Firmware Upgrade 16-20 nodes", 25 => "Node Firmware Upgrade 21-25 nodes", 30 => "Node Firmware Upgrade 26-30 nodes", # 35 => "Node Firmware Upgrade 31-35 nodes", # 40 => "Node Firmware Upgrade 36-40 nodes", # 45 => "Node Firmware Upgrade 41-45 nodes", # 50 => "Node Firmware Upgrade 46-50 nodes", # 55 => "Node Firmware Upgrade 51-55 nodes", # 60 => "Node Firmware Upgrade 56-60 nodes", }, 'Rolling Node Upgrade' => { 5 => "Rolling 3 to 5 Node Upgrade", 10 => "Rolling 6 to 10 Node Upgrade", 20 => "Rolling 11 to 20 Node Upgrade", 30 => "Rolling 21 to 30 Node Upgrade", 40 => "Rolling 31 to 40 Node Upgrade", # 50 => "Rolling 41 to 50 Node Upgrade", # 60 => "Rolling 51 to 60 Node Upgrade", # 70 => "Rolling 61 to 70 Node Upgrade", # 80 => "Rolling 71 to 80 Node Upgrade", # 90 => "Rolling 81 to 90 Node Upgrade", }, 'Rolling Reboot' => { 10 => "Rolling 3-10 Reboot", 20 => "Rolling 11-20 Reboot", 30 => "Rolling 21-30 Reboot", 40 => "Rolling 31-40 Reboot", 50 => "Rolling 41-50 Reboot", 60 => "Rolling 51-60 Reboot", 70 => "Rolling 61-70 Reboot", 80 => "Rolling 71-80 Reboot", # 90 => "Rolling 81-90 Reboot", }, 'FCO F042415EE' => { 5 => "FCO F042415EE Drive Firmware Upgrade 1-5 nodes", 10 => "FCO F042415EE Drive Firmware Upgrade 6-10 nodes", 15 => "FCO F042415EE Drive Firmware Upgrade 11-15 nodes", 20 => "FCO F042415EE Drive Firmware Upgrade 16-20 nodes", 25 => "FCO F042415EE Drive Firmware Upgrade 21-25 nodes", }, 'Isilon Hourly' => { 1 => "Isilon 1 Hour", 1.5 => "Isilon 1.5 Hours", 2 => "Isilon 2 Hours", 2.5 => "Isilon 2.5 Hours", 3 => "Isilon 3 Hours", 3.5 => "Isilon 3.5 Hours", 4 => "Isilon 4 Hours", 4.5 => "Isilon 4.5 Hours", 5 => "Isilon 5 Hours", 5.5 => "Isilon 5.5 Hours", 6 => "Isilon 6 Hours", 6.5 => "Isilon 6.5 Hours", 7 => "Isilon 7 Hours", 7.5 => "Isilon 7.5 Hours", 8 => "Isilon 8 Hours", 8.5 => "Isilon 8.5 Hours", 9 => "Isilon 9 Hours", 9.5 => "Isilon 9.5 Hours", 10 => "Isilon 10 Hours", 10.5=> "Isilon 10.5 Hours", 11 => "Isilon 11 Hours", 11.5=> "Isilon 11.5 Hours", 12 => "Isilon 12 Hours", }, ); } # Code Links BEGIN { our %CodeLinks = ( # Node Firmware Packages "Isilon Node Firmware Package 10.2.1" => { "Code" => "https://download.emc.com/downloads/DL89290_Isilon_Node_Firmware_Package_10.2.1.tar", "sha256" => "32c479d97385cd5f3af82a067e3ede1b2b8930b4a0e570267aafba4dbde8b28e", }, "Isilon Node Firmware Package 10.1.6" => { "Code" => "ftp://ftp.emc.com/pub/rcm/Isilon/firmware/IsiFw_Package_v10.1.6.tar", "sha256" => "eed9035b029b63d459007d728e306de8382d09b4446661de68b32761062cbcd6", }, "Isilon Node Firmware Package 10.1.3" => { "sha256" => "dc8edd4005a813e472aeda6768bc540a9cccd93eca1cf52678436c3697436156", }, "Isilon Node Firmware Package 10.1.2" => { "sha256" => "22140a619c13567a04fccea9b45da4ce776a4450b158a8278bb6c86cb25183b8", }, "Isilon Node Firmware Package 10.1.1" => { "sha256" => "8b0a171feb1ab68e82309d6b7498d3cfe79550d4b79cee8375d1816bd0d3ff3f", }, "Isilon Node Firmware Package 10.0.1" => { "sha256" => "54693be0b17f84a87725acfc8ccc5a07319b182b220f7a88e9f3953444c11d94", }, "Isilon Node Firmware Package 9.3.5" => { "Code" => "https://download.emc.com/downloads/DL83480_Isilon_Node_Firmware_Package_9.3.5.tar", "sha256" => "f03d11fd5d8e909b7a71e6e2393234cc3a35599cca056dc35aca0fe72232dcf9", }, "Isilon Node Firmware Package 9.3.4" => { "sha256" => "1bf457fb290cda2e6113bfa9e17b874ea62e54c979e0abd3a773db58357ff43f", }, "Isilon Node Firmware Package 9.3.3" => { "sha256" => "17a6707e048168f58933627e81eb09d8d31d7e10179018039c6cddab55b58920", }, # BMC Firmware Packages "Isilon BMC Firmware Package 1.0" => { "Code" => "ftp://ftp.emc.com/pub/rcm/Isilon/firmware/IsiFw_Package_BMC_v1.0.tar", "sha256" => "144222e82d594b9239873d32bbbfaa1f2557e4031196f0b0d8141f2860e4bc3b", }, # Drive Firmware Packages "Isilon Drive Firmware Package 1.18" => { "Code" => "https://download.emc.com/downloads/DL83529_Isilon_Drive_Firmware_Package_1.18.tgz", "sha256" => "6d50434fbc2f0e39f430fb44360dfe3af173a40ff2006bef58df06d3e9ed6e97", }, # Drive Support Packages "Isilon Drive Support Package 1.29" => { "Code" => "https://download.emc.com/downloads/DL92292", "sha256" => "07e6b723f60470b08159f3fa27fdbfc39c211afcd6577536ccbf1114f9846ce4", }, # FRU Package "IsiFru Package" => { "Code" => "ftp://ftp.emc.com/outgoing/Fru_Package/IsiFru_Package_201810171449.tgz", "sha256" => "3cc8f0e6538240ff32bb04b845b3dd92f516ad7ed1139afd8bd0c9bf6e6b04eb", }, # Scripts "NFS Export Security Repair Script" => { "Code" => "ftp://ftp.emc.com/outgoing/support/isi_nfs_export_security_repair_script/isi_nfs_export_security_repair", "sha256" => "7d5a18a47f10b6a326be83e141a28cad1f7c14ed2098d741d4b12d41963b9f54", }, "Gen6 Power Supply Check Script" => { "Code" => 'ftp://ftp.emc.com/outgoing/KB%20Support%20Files/KB%20519119/gen6_ps_check.py', "sha256" => "3da33d6bd42d09ae4bbd8309e5afef0a424d0eb75fc5d8bec6ee4464eed57301", }, # Patches "HealthCheck_8.1.2_2019-01" => { "Code" => "https://download.emc.com/downloads/DL92231", "sha256" => "789c4885b7fce5feff75e85fc1290413e463e408fe2fbbb7a571adbdb820fbed", }, "HealthCheck_8.1.0_v2.0" => { "Code" => "https://download.emc.com/downloads/DL91612_Isilon_Health_Check_SP2_for_OneFS_8.1.0.tgz", "sha256" => "ed55677e093599612d6e4b5944505745fcce80aacd1f03a7c0c1c8920b383a2e", }, "HealthCheck_8.0.0.7_v2.0" => { "Code" => "https://download.emc.com/downloads/DL91611_Isilon_Health_Check_SP2_for_OneFS_8.0.0.7.tgz", "sha256" => "c3b399d66502719b8a43141a7113dd0874a6a808979e942a18c8e8c737d19548", }, "patch-243897" => { "Code" => "https://download.emc.com/downloads/DL92220", "sha256" => "edeae9b0680484200b9657b372c6d3c004b6be74e490bfce5b84dd1e7e50ec0f", }, "patch-243894" => { "Code" => "https://download.emc.com/downloads/DL92219", "sha256" => "3d97b4790c939c6e5465a5e6388a77541e670f91366b0e65b22b0f1fc9bbe273", }, "patch-242570" => { "Code" => "https://download.emc.com/downloads/DL92096", "sha256" => "57f8738c80bfa895530a819eb11185c074ad3f933cb72de30d965293a1d2e453", }, "patch-241584" => { "Code" => "https://download.emc.com/downloads/DL92095", "sha256" => "62b847dabc3a424ca155e8760fca51f8b1dddf8daab07bc5a272850f89638f2c", }, "patch-240795" => { "Code" => "https://download.emc.com/downloads/DL92147", "sha256" => "6f3b0b184ea528219efe795300978bc6b9799a3b70be91d537cef3525dcff250", }, "patch-239649" => { "Code" => "https://download.emc.com/downloads/DL91315_Isilon_OneFS_Patch-239649.tgz", "sha256" => "cd5f8ef00a74165cefeb182dc442adf8b69379cc9dbc8fdb73b65e4d0490aa0d", }, "patch-233663" => { "Code" => "https://download.emc.com/downloads/DL90991_Isilon_OneFS_Patch-233663.tgz", "sha256" => "c3a7902109721bca762a9e90c83b52c62e5e719c2eb042b6d01f48812a18d5d4", }, "patch-231386" => { "Code" => "https://download.emc.com/downloads/DL90411_Isilon_OneFS_Patch-231386.tgz", "sha256" => "84276a3f1c2a97aa9bc784944bda2229032c4df00e1471691898eb4e7240b3c8", }, "patch-229160" => { "Code" => "https://download.emc.com/downloads/DL90208_Isilon_OneFS_Patch-229160.tgz", "sha256" => "9b20e01f7a87b1950c51399c11c2c9f805c50cc95d6752f401836ae10f58cafe", }, "patch-229158" => { "Code" => "https://download.emc.com/downloads/DL90209_Isilon_OneFS_Patch-229158.tgz", "sha256" => "fd8bce193555161b1bf0522768463e74229e37f7b188be90aa10007a57ae5185", }, "patch-229156" => { "Code" => "https://download.emc.com/downloads/DL90210_Isilon_OneFS_Patch-229156.tgz", "sha256" => "de4694c080ffb3048f1208175cc492cf426ffc0891f8ff59d0eb7948cfbfe79b", }, "patch-228458" => { "Code" => "https://download.emc.com/downloads/DL90409_Isilon_OneFS_Patch-228458.tgz", "sha256" => "2ba8ab8a5ab4e28e4e69b3b16c1719146e6a50ac81971c4466f0e522c5d6f826", }, "patch-228202" => { "Code" => "https://download.emc.com/downloads/DL91531_Isilon_OneFS_Patch-228202.tgz", "sha256" => "3853fa1e0162e5670effd369e9fddc28fa0b6a627f734d8ea56e9600a4f86ac0", }, "patch-226984" => { "Code" => "https://download.emc.com/downloads/DL89610_Isilon_OneFS_Patch-226984.tgz", "sha256" => "932ad12392556271217c7099fd0dca4eb3b87094f80a0bed0797e0361ab26b74", }, "patch-226323" => { "Code" => "https://download.emc.com/downloads/DL89668_Isilon_OneFS_Patch-226323.tgz", "sha256" => "724032a9f2bcafdd9a2027b5ffef6561340a1e1720524b9e31822c95c7ff5748", }, "patch-224399" => { "Code" => "https://download.emc.com/downloads/DL89135_Isilon_OneFS_Patch-224399.tgz", "sha256" => "d36b4ea44d5d793930bb4cbbabc10a9d1ccc313ab380548dea3b4d4304a1206f", }, "patch-224116" => { "Code" => "https://download.emc.com/downloads/DL90410_Isilon_OneFS_Patch-224116.tgz", "sha256" => "9420b627b7dd419b93a7fed275948e1414640eaee9a35b2ad73238d38c5bf72f", }, "patch-223338" => { "Code" => "https://download.emc.com/downloads/DL90474_Isilon_OneFS_Patch-223338.tgz", "sha256" => "faf53ca44e03292266f00e211409acb2076c000bdc5ee89d3010ad1ef35dfb0a", }, "patch-223311" => { "Code" => "https://download.emc.com/downloads/DL90081_Isilon_OneFS_Patch-223311.tgz", "sha256" => "fa11cb91d73266a22fc9950cba85715ec7e2d72d16e4734d353cc51f337d92c6", }, "patch-222487" => { "Code" => "https://download.emc.com/downloads/DL89176_Isilon_OneFS_Patch-222487.gz", "sha256" => "b1e9ecafb4f793cd00e84947b5e6672bc60d984eb5501f26d01f7872ea3ee106", }, "patch-221949" => { "Code" => "https://download.emc.com/downloads/DL89175_Isilon_OneFS_Patch-221949.gz", "sha256" => "5e280e5bfaa877e8217418128d82553ad60787e037dd1caef79403fcc1e571a1", }, "patch-221947" => { "Code" => "https://download.emc.com/downloads/DL89667_Isilon_OneFS_Patch-221947.tgz", "sha256" => "bf3a53f4b490bcc67faf07fbd0fe84d01f199e74f892cde46978eeca9de5c27b", }, "patch-221946" => { "Code" => "https://download.emc.com/downloads/DL88631_Isilon_OneFS_Patch-221946.tgz", "sha256" => "ac14bd365452176a4c2b25f5dbd1fbaa1415a4da5f352d5566b89a597ad8b8c2", }, "patch-221638" => { "Code" => "https://download.emc.com/downloads/DL89137_Isilon_OneFS_Patch-221638.tgz", "sha256" => "21f0a7d02d619c9f0f9ea9dda9b03d2bf3afa1d812c909471857ef49644a222a", }, "patch-221617" => { "Code" => "https://download.emc.com/downloads/DL88630_Isilon_OneFS_Patch-221617.tgz", "sha256" => "e0e0b4a80a499dcb8d7c2f66b2aecdeb89a0b9aef9ba0603eeaeea7ee188dced", }, "patch-221603" => { "Code" => "https://download.emc.com/downloads/DL89174_Isilon_OneFS_Patch-221603.gz", "sha256" => "d5164a061d275713a4ce899ab83ef595aba49470289e7ae3d07f0f5dae68b131", }, "patch-221602" => { "Code" => "https://download.emc.com/downloads/DL88628_Isilon_OneFS_Patch-221602.tgz", "sha256" => "3e9b8b8c79bd24912f22ae52fa27c4b076f924d486a22f180c969fdc3116717a", }, "patch-220176" => { "Code" => "https://download.emc.com/downloads/DL89173_Isilon_OneFS_Patch-220176.gz", "sha256" => "5d7a367db9ce02c9e0816db232dc127646a05eb232a61c7208c806617d409662", }, "patch-220098" => { "Code" => "https://download.emc.com/downloads/DL89485_Isilon_OneFS_Patch-220098.tgz", "sha256" => "cb5cdcd1cad22828e722eb0f1a163e9f67fa57271c908ed1d7b5820bdfc9363f", }, "patch-219279" => { "Code" => "https://download.emc.com/downloads/DL89676_Isilon_OneFS_Patch-219279.tgz", "sha256" => "f5e86c749d1de7fcc0a9b8ee56cb1dc79bf2e71b90cb07fa3a916d6a6e728b9f", }, "patch-217638" => { "Code" => "https://download.emc.com/downloads/DL88034_Isilon_OneFS_Patch-217638.tgz", "sha256" => "0be46fabcf5e4c43f0623bdb64a2502a58eab8615590817e5708c8cf0124cb74", }, "patch-217637" => { "Code" => "https://download.emc.com/downloads/DL87915_Isilon_OneFS_Patch-217637.tgz", "sha256" => "219c8b0389d63313d59e3eebba96bcc45f34a8b2f1f638ba1ca239052b39d99c", }, "patch-213287" => { "Code" => "https://download.emc.com/downloads/DL88627_Isilon_OneFS_Patch-213287.tgz", "sha256" => "9c460a6ec57fbea5f7b369f8a31b97f5494b370bc97ee000997c36804d9eb5fc", }, "patch-213286" => { "Code" => "https://download.emc.com/downloads/DL88626_Isilon_OneFS_Patch-213286.tgz", "sha256" => "5c537b089a2da896328e5a8c8d4930952adeb2d68063468ee71fff7ecd75fbd1", }, "patch-213281" => { "Code" => "https://download.emc.com/downloads/DL88171_Isilon_OneFS_Patch-213281.tgz", "sha256" => "61591a2f2c51c8c726e1d2f0c4d40bbfec12ceb78cc02ec8851d3bdfbe79f806", }, "patch-213280" => { "Code" => "https://download.emc.com/downloads/DL88003_Isilon_OneFS_Patch-213280.tgz", "sha256" => "a9bc5f40f689c33b59a50641c7fcbab0fa9b54a39f74ac83217189092c1b2603", }, "patch-213279" => { "Code" => "https://download.emc.com/downloads/DL88249_Isilon_OneFS_Patch-213279.tgz", "sha256" => "13191be36b27be6e08b9b5fc44624b351241edd490aa0bb111290047226f9260", }, "patch-211361" => { "Code" => "https://download.emc.com/downloads/DL87018_Isilon_OneFS_Patch-211361.tgz", "sha256" => "14c96e73af2dc9e39c62f93ce7ab004cda3b66a9d2d133877473d0aeffc1b187", }, "patch-206322" => { "Code" => "https://download.emc.com/downloads/DL86823_Isilon_OneFS_Patch-206322.tgz", "sha256" => "6d99bfa5ee6f28b14871b6e2f5f5ac6b5a9122c3aefa15f73a609d526426fc32", }, "patch-189133" => { "Code" => "https://download.emc.com/downloads/DL83560_Isilon_OneFS_Patch-189133.tgz", "sha256" => "0bd5e5e8e1576483549bfd80195c1e578b07b195ff09c6106cfae87acfc59764", }, "patch-189132" => { "Code" => "https://download.emc.com/downloads/DL83559_Isilon_OneFS_Patch-189132.tgz", "sha256" => "12ad326888591a737ec394bc620f5401c563dc1bc3d76ba946e0ef898acc3731", }, "patch-188660" => { "Code" => "https://download.emc.com/downloads/DL82519_Isilon_OneFS_Patch-188660.tgz", "sha256" => "38285bce926eb33bf6ce2c0a314ecaec8712b58c44c29c133072a18d72bbe3e4", }, "patch-188239" => { "Code" => "https://download.emc.com/downloads/DL82813_Isilon_OneFS_Patch-188239.tgz", "sha256" => "2709ef574256096358625d0baec77723029d43bde4d65ea51c94b8e37c9ecbdf", }, "patch-181490" => { "Code" => "https://download.emc.com/downloads/DL80015_Isilon_OneFS_Patch-181490.tgz", "sha256" => "a8e44b68f40a7d4c3e21ec3682e81e0d8b89001aacd6222cd93581c123fd2e05", }, "patch-179385" => { "Code" => "ftp://ftp.emc.com/pub/rcm/Isilon/patch/patch-179385.tgz", "sha256" => "9ca3506199f516862a8791a10c0ace006558336639e0d4bda3c1c00952298e5b", }, "patch-178793" => { "Code" => "ftp://ftp.emc.com/pub/rcm/Isilon/patch/patch-178793.tgz", "sha256" => "0aa99678272b611a6608bd4564b04831cfe28f62b59687c0d645b5afb8fbc88f", }, "patch-175451" => { "Code" => "https://download.emc.com/downloads/DL71573_Isilon_OneFS_Patch-175451.tgz", "sha256" => "8318d05b00f00e01a0b916da9ca912b2b03ffba2cad6f5a939eb18d2c49fbf61", }, "patch-164666" => { "Code" => "https://download.emc.com/downloads/DL70637_Isilon_OneFS_Patch-164666.tgz", "sha256" => "25b53e98b9caf89c3b5866f8caf46b2e6d0b29148c3c53132fdeb4fa0afec8cf", }, "patch-161166" => { "Code" => "https://download.emc.com/downloads/DL61739_Isilon_OneFS_Patch-161166.tgz", "sha256" => "39614c55cbe5f4f9eb536b23bc67b411f07c36b65142f5951d55be8281ab31bf", }, "patch-147686" => { "Code" => "https://download.emc.com/downloads/DL59058_Isilon_OneFS_Patch-147686.tgz", "sha256" => "e934289d49d2f722be22e4d750b5f78e4584f95b7cea582ec03f5245374317c9", }, "patch-146974" => { "Code" => "https://download.emc.com/downloads/DL58909_Isilon_OneFS_Patch-146974.tgz", "sha256" => "28fea8daea8e3588cce3b263a72cdfa49ebdfa553826483eb1688d65590ecd61", }, "patch-141264" => { "Code" => "https://download.emc.com/downloads/DL58028_Isilon_OneFS_Patch-141264.tgz", "sha256" => "3bd4a7268c0d2b14475567ed7e496beab7664ec98bc7a833b5dd9f6a7269a044", }, "patch-141263" => { "Code" => "https://download.emc.com/downloads/DL58027_Isilon_OneFS_Patch-141263.tgz", "sha256" => "7bb9fc6ca8d8a3133623da45cfdd21942082e37d9be156cbf3205f38717623a1", }, "patch-141262" => { "Code" => "https://download.emc.com/downloads/DL58026_Isilon_OneFS_Patch-141262.tgz", "sha256" => "b2d9d3ae0b7b6aef3f83c08320023ae2f91a99caa0d30fd3d6df257fb9721eda", }, "patch-134049" => { "Code" => "https://download.emc.com/downloads/DL55811_Isilon_OneFS_Patch-134049.tgz", "sha256" => "544ea0986deb55e0142c2d4b33fed99292b59f6b0f357cea0c5905776527fdf3", }, "patch-130491" => { "Code" => "https://download.emc.com/downloads/DL55310_Isilon_OneFS_Patch-130491.tgz", "sha256" => "66ff73447d6041261f3d529b923b25b9bdd50b3533e63c0fb482a8c4359f5074", }, # OneFS codes "Isilon OneFS 8.1.3.0 installation file" => { "Code" => "Directed Availability Release, approval required and file must be provided via temporary FTP link.", "sha256" => "5c02c4165956565b5a32009b9752bf7270920c5538a7ff210991338096b5e306", }, "Isilon OneFS 8.1.2.0 installation file" => { "Code" => "https://download.emc.com/downloads/DL90378_Isilon_OneFS_8.1.2.0_installation_file.tar.gz", "sha256" => "f2de544f6616cc681a7889090e5970a1ea3363f9e9e44a4fd1ca1b45ea9b3a3e", }, "Isilon OneFS 8.1.1.1 installation file" => { "Code" => "Directed Availability Release, approval required and file must be provided via temporary FTP link.", "sha256" => "b71624012ad5707673d0eb013ef9b3b26a2b221d4f93477f0d097e98e99e071f", }, "Isilon OneFS 8.1.1.0 installation file" => { "Code" => "Directed Availability Release, approval required and file must be provided via temporary FTP link.", "sha256" => "a3c31d5bc4cad1dfda98128483f9bef5247c240a7e45afa27ff9047e6eca0d5d", }, "Isilon OneFS 8.1.0.4 installation file" => { "Code" => "https://download.emc.com/downloads/DL89606_Isilon_OneFS_8.1.0.4_Installation_File.tar.gz", "sha256" => "923444b5901340f3a9efe842ae974ba830492c4d26e0f12607f67b8d8f2762ab", }, "Isilon OneFS 8.1.0.3 installation file" => { "Code" => "Monitored Release, file must be provided via temporary FTP link.", "sha256" => "c09fddaeb0cdb288343bd4fda17c9055d33baa65196f5c99fc86c701cc7dc210", }, "Isilon OneFS 8.1.0.2 installation file" => { "Code" => "Monitored Release, file must be provided via temporary FTP link.", "sha256" => "06aec34e776de15b684aa7c0756dcf526b372f4bb7239e8a31aadf4b9a9251dd", }, "Isilon OneFS 8.1.0.1 installation file" => { "Code" => "Monitored Release, file must be provided via temporary FTP link.", "sha256" => "0abf77833b029f4973a5c0f84d03868be16a0aa8653ff92370d8a6d5701ed237", }, "Isilon OneFS 8.1.0.0 installation file" => { "Code" => "Monitored Release, file must be provided via temporary FTP link.", "sha256" => "17cbdbad1af10d1ec395f4d25b4c468c84224c17e6fe41c37752e7f426b8fd80", }, "Isilon OneFS 8.0.1.2 installation file" => { "Code" => "https://download.emc.com/downloads/DL86934_Isilon_OneFS_8.0.1.2_installation_file.tar.gz", "sha256" => "9115634b5fdbe25b2f8569cc52ad23cafa2a3ad60ab75fb320a715fdf912731a", }, "Isilon OneFS 8.0.1.1 installation file" => { "Code" => "https://download.emc.com/downloads/DL83292_Isilon_OneFS_8.0.1.1_installation_file.tar.gz", "sha256" => "ae4a45990ad8858e2707718eb5395df8e8d121fda3ae66b1b7f4a1e654415b72", }, "Isilon OneFS 8.0.1.0 installation file" => { "Code" => "https://download.emc.com/downloads/DL79685_OneFS_8.0.1.0_Installation_File.gz", "sha256" => "2756dfa3570e12fc13f6d97b4ba0f8c8c2c2bd21b81bdc7b277feec6381b89d1", }, "Isilon OneFS 8.0.0.7 installation file" => { "Code" => "https://download.emc.com/downloads/DL89995_Isilon_OneFS_8.0.0.7_installation_file.tar.gz", "sha256" => "64f7ccdc2506a631b0d0e2662df06420dd2c56ed2edf0faee1fb2eb8ac1fce0f", }, "Isilon OneFS 8.0.0.6 installation file" => { "Code" => "https://download.emc.com/downloads/DL87158_Isilon_OneFS_8.0.0.6_installation_file.tar.gz", "sha256" => "52b5fa36feda644fc00a88be2320d00cd3033b3a280d4a2d49001b3153c831fc", }, "Isilon OneFS 8.0.0.5 installation file" => { "Code" => "https://download.emc.com/downloads/DL84866_Isilon_OneFS_8.0.0.5_Installation_File.tar.gz", "sha256" => "707f63afcd66502e80e0e0921254bfc3c1f5ef573ccfa42a93e880bd34321a34", }, "Isilon OneFS 8.0.0.4 installation file" => { "Code" => "https://download.emc.com/downloads/DL82429_Isilon_OneFS_8.0.0.4_installation_file.tar.gz", "sha256" => "b2462be524ef9a5f3921d4ffc5438da9e2d54aad27aa222847ac83daecbad75d", }, "Isilon OneFS 8.0.0.3 installation file" => { "Code" => "https://download.emc.com/downloads/DL80176_Isilon_OneFS_8.0.0.3_installation_file.tar.gz", "sha256" => "d0e7888d0ab0388ec427461c5dc9e0c52d5b34897949f21120b7ae40c07fe842", }, "Isilon OneFS 8.0.0.2 installation file" => { "Code" => "https://download.emc.com/downloads/DL78319_Isilon_OneFS_8.0.0.2_installation_file.tar.gz", "sha256" => "ccc46fea2960d625ded3e1664b96503c0e7c9f060aa921af42a18b70d06e9ab8", }, "Isilon OneFS 8.0.0.1 installation file" => { "Code" => "https://download.emc.com/downloads/DL70277_Isilon_OneFS_8.0.0.1_installation_file.gz", "sha256" => "3c1a27a9237c35d24934452beb07080d6836ae930465b4afc1d83695feea721d", }, "Isilon OneFS 7.2.1.6 installation file" => { "Code" => "https://download.emc.com/downloads/DL83967_Isilon_OneFS_7.2.1.6_installation_file.gz", "sha256" => "47d6f772ad1bf69ae9aaeb1443ae21f235d30b4e53a594412575167202b76185", }, "Isilon OneFS 7.2.1.5 installation file" => { "Code" => "https://download.emc.com/downloads/DL81748_Isilon_OneFS_7.2.1.5_Installation_File.gz", "sha256" => "47bbf32382a510663b33e1a21fbc29ea29134dd2fed96b1018ccd6138295438a", }, "Isilon OneFS 7.2.1.4 installation file" => { "Code" => "https://download.emc.com/downloads/DL78874_Isilon_OneFS_7.2.1.4_installation_file.gz", "sha256" => "ed86d12527883c0dae5dfa6b8362a4a253bb3cdd6e2725bb5e6489a63daa56f8", }, "Isilon OneFS 7.2.1.3 installation file" => { "Code" => "https://download.emc.com/downloads/DL70734_Isilon_OneFS_7.2.1.3_installation_file.gz", "sha256" => "3112884270cd083721e73a1ca618b66ffa1d938885af5b8b9a879cdbbd82129e", }, "Isilon OneFS 7.2.1.2 installation file" => { "Code" => "https://download.emc.com/downloads/DL66060_Isilon_OneFS_7.2.1.2_installation_file.tgz", "sha256" => "ae6e6c0d0ce63644c44adc04f583bf3c249e95053961e27984482d58817a4e62", }, "Isilon OneFS 7.2.0.5 installation file" => { "Code" => "https://download.emc.com/downloads/DL62853_Isilon_OneFS_7.2.0.5_installation_file.tar.gz", "sha256" => "2c195e27425e87543c188754f086a01a28188bbba40bf8860b9bc68db558a54d", }, "Isilon OneFS 7.2.0.4 installation file" => { "Code" => "https://download.emc.com/downloads/DL61245_Isilon_OneFS_7.2.0.4_installation_file.tar.gz", "sha256" => "7fec49199fe78b1f4cd3fe06d71a0ff5329099a3c9c66ccbe5c93134c67be436", }, "Isilon OneFS 7.1.1.11 installation file" => { "Code" => "ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v7.1.1.11_Install.tar.gz", "sha256" => "37d7dd688a9f6c038418a5b2959ff5f8fc490d5429eab605a3cc17d3e541502c", }, ); } # Activity durations # Base -> Time required to validate checksum and complete and pre/post # steps such as installing/uninstalling a firmware package # Drives/Nodes -> Time required per drive or node for updates to be performed # Regex Expression -> Used to add additional time based on regex criteria # These must be accomodated for in the getActivityDuration routine BEGIN { our %Durations = ( 'Pre-check' => 20, 'Post-check' => 20, 'Drive Firmware' => { "Base" => 15, "Nodes" => 10, "Drives" => 0.4, }, 'Drive Firmware Package' => { "Base" => 15, "Nodes" => 10, }, 'Drive Support Package' => { "Base" => 15, "Nodes" => 10, "Drives" => 0.4, }, 'Node Firmware' => { "Base" => 15, "Nodes" => 10, '^(A200|A2000|H400|H500|H600|F800)' => 25, '^(CMC|CMD|CAR)' => 7, }, 'BMC Firmware' => { "Base" => 15, "Nodes" => 10, }, 'Rolling OneFS' => { "Base" => 25, "Nodes" => 15, }, 'Simultaneous OneFS' => { "Base" => 45, }, 'Rolling Reboot' => { "Base" => 15, "Nodes" => 8, }, 'Simultaneous Reboot' => { "Base" => 25, }, 'Reboot' => { '^(S210|X410|X210|NL410)' => 3, '^(HD400)' => 10, }, 'Patch Install (Rebootless)' => { "Base" => 15, "Patch" => 15, } ); } $CHECKS{'checkDriveFirmware'}{'Description'} = "Checks for out of date Drive Firmware and calls other related drive firmware checks"; sub checkDriveFirmware { return if wasCalled(); checkDriveSupportPackage(); checkDriveFirmwarePackage(); checkFCOF042415EE(); checkKB469133(); } # End checkDriveFirmware $CHECKS{'checkDriveFirmwarePackage'}{'Description'} = "Checks for drive firmware updates available in the Drive Firmware Package"; sub checkDriveFirmwarePackage { return if wasCalled(); # Skip if no valid firmware version to check against return unless $RPS{'DFP'} and $RPS{'DFP'} ne "None"; # Skip if the cluster is virtual gatherProductInfo(); return if $CLUSTER_INFO{'Virtual'}; # Skip if there are no IMDD nodes checkPartitions(); return unless $CLUSTER_INFO{'IMDD'}; printCheck("Drive Firmware Package ($RPS{'DFP'})"); gatherCTO(); gatherDriveFirmware(); gatherPatches(); our %DriveFirmware; # Analyze for out of date firmware foreach my $model (keys %{$CLUSTER_INFO{'Drive Firmware'}}) { foreach my $firmware (keys %{$CLUSTER_INFO{'Drive Firmware'}{$model}}) { next unless $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; my $fw = $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; my $dfp_target = 0; ### # Check for drive firmware updates if ($DriveFirmware{$RPS{'DFP'}}{$model} and $DriveFirmware{$RPS{'DFP'}}{$model} ne $firmware and $firmware !~ /^SN0[46]$/) { # Drive firmware updates available $dfp_target = $DriveFirmware{$RPS{'DFP'}}{$model}; } ### # Check for nodes that only support DFP foreach my $lnn (keys %{$fw->{'Counts'}}) { # Only check on IMDD nodes next unless $NODE_INFO{$lnn}{'IMDD'}; # Add to the list of nodes push(@{$fw->{'DFP'}{'Nodes'}}, $lnn); # Add to the count of devices $fw->{'DFP'}{'Count'} += $fw->{'Counts'}{$lnn}; # Flag FRU Package if any CTO nodes require updating if ($dfp_target and $NODE_INFO{$lnn}{'CTO'}) { $RPS{'Drive Firmware Package'}{'FRU'} = 1; } } # Next firmware if no updates were found next unless $dfp_target and $fw->{'DFP'}{'Nodes'}; ### # Beyond this point drive firmware updates are available. debug("checkDriveFirmwarePackage: ". "DFP update available in $RPS{'DFP'}: ". "$firmware has newer firmware available: $dfp_target"); # Set target firmware version $fw->{'DFP'}{'Target'} = $dfp_target; # Add to the list of nodes to be updated @{$RPS{'Drive Firmware Package'}{'Nodes'}} = merge(@{$RPS{'Drive Firmware Package'}{'Nodes'}}, @{$fw->{'DFP'}{'Nodes'}}); } } # Print out drive firmware information details(sprintf("%-30s %-10s %-10s %-5s %s", "Model", "Firmware", "DFP ($RPS{'DFP'})", "Count", "Nodes")); foreach my $model (keys %{$CLUSTER_INFO{'Drive Firmware'}}) { foreach my $firmware (keys %{$CLUSTER_INFO{'Drive Firmware'}{$model}}) { my $fw = $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; next unless ($fw->{'DFP'}{'Nodes'} and $fw->{'DFP'}{'Count'}); details(sprintf("%-30s %-10s %-10s %-5s %s", $model, $firmware, ($fw->{'DFP'}{'Target'} or "-"), $fw->{'DFP'}{'Count'}, compressRange(@{$fw->{'DFP'}{'Nodes'}}))); } } if (!$RPS{'Drive Firmware Package'}{'Nodes'}) { # Drive firmware updates not available printResults(); return; } info(scalar(@{$RPS{'Drive Firmware Package'}{'Nodes'}}). " nodes have out of date drive firmware: ". compressRange(@{$RPS{'Drive Firmware Package'}{'Nodes'}})); # Check if a drive support patch is required my $patch = ""; if (!hasPatch("patch-(134049|130491|14126[234])")) { # Different patch number based on current OneFS version $_ = $CLUSTER_INFO{'OneFS'}; if (/^6\.5\.4/) { $patch = 'patch-134049'; } elsif (/^6\.5\.5/) { $patch = 'patch-130491'; } elsif (/^7\.0\.1/) { $patch = 'patch-141264'; } elsif (/^7\.0\.2/) { $patch = 'patch-141263'; } elsif (/^7\.1\.0/) { $patch = 'patch-141262'; } elsif (/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/) { } elsif (/^7|^8/) { $patch = "Isilon Drive Support Package $RPS{'DSP'}" if $RPS{'DSP'} and $RPS{'DSP'} ne "None"; } } if ($patch) { # Flag if a drive support patch is required $RPS{'Drive Firmware Package'}{'Patch'}{$patch}{'Impacts'} = ""; info("The drive support patch, $patch, is required"); } if ($RPS{'Drive Firmware Package'}{'FRU'}) { # Flag if the FRU package is required info("IsiFru Package is required"); } # Drive firmware updates using Drive Firmware Package can be performed # regardless of the current OneFS version, always set to position 1. $RPS{'Drive Firmware Package'}{'Position'} = 1; $RPS{'Drive Firmware Package'}{'Target'} = "Drive Firmware Package $RPS{'DFP'}"; # Append files required for the updates push(@{$RPS{'Drive Firmware Package'}{'Files'}}, "Isilon $RPS{'Drive Firmware Package'}{'Target'}") if $RPS{'Drive Firmware Package'}{'Target'}; push(@{$RPS{'Drive Firmware Package'}{'Files'}}, "IsiFru Package") if $RPS{'Drive Firmware Package'}{'FRU'}; # Flag reasons why drive firmware updates should be a priority if ($RPS{'ETA'}{'193819'}{'Nodes'}) { my @nodes = (); foreach my $lnn (@{$RPS{'ETA'}{'193819'}{'Nodes'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Firmware Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). " nodes are affected by ETA 193819: ". compressRange(@nodes); warning($message); push(@{$RPS{'Drive Firmware Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } if ($RPS{'Drive Firmware'}{'KB88930'}) { my @nodes = (); foreach my $lnn (@{$RPS{'Drive Firmware'}{'KB88930'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Firmware Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). " nodes are affected by KB88930: ". compressRange(@nodes); warning($message); push(@{$RPS{'Drive Firmware Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } if ($RPS{'Drive Firmware'}{'KB502084'}) { my @nodes = (); foreach my $lnn (@{$RPS{'Drive Firmware'}{'KB502084'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Firmware Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). " nodes are affected by KB502084: ". compressRange(@nodes); warning($message); push(@{$RPS{'Drive Firmware Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } # Remove Drive Firmware Updates if requested only priority updates and there are no priority updates needed. if ($RPS{'Priority DF Only'} and $RPS{'Drive Firmware Package'}{'Nodes'} and !$RPS{'Drive Firmware Package'}{'Priority'}) { delete $RPS{'Drive Firmware Package'}; } info("Refer to KB210512 (https://support.emc.com/kb/210512) for details.") if (@INFO or @WARN or @FAIL); printResults(); } # End checkDriveFirmwarePackage $CHECKS{'checkDriveSupportPackage'}{'Description'} = "Checks for drive firmware updates available in the Drive Support Package"; sub checkDriveSupportPackage { return if wasCalled(); # Skip if no valid firmware version to check against return unless $RPS{'DSP'} and $RPS{'DSP'} ne "None"; # Skip if the cluster is virtual gatherProductInfo(); return if $CLUSTER_INFO{'Virtual'}; # Skip if there are no Bootflash nodes checkPartitions(); return unless $CLUSTER_INFO{'Bootflash'}; printCheck("Drive Support Package ($RPS{'DSP'})"); gatherCTO(); gatherDriveFirmware(); our %DriveSupport; # Generate a sorted list of drive support packages my @dsps = sort {compareVersions($b, $a)} keys %DriveSupport; # Analyze for out of date firmware foreach my $model (keys %{$CLUSTER_INFO{'Drive Firmware'}}) { foreach my $firmware (keys %{$CLUSTER_INFO{'Drive Firmware'}{$model}}) { next unless $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; my $fw = $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; my $dsp_target = 0; foreach my $pkg (@dsps) { # Stop if we found a target last if $dsp_target; # Skip this package if no updates for this model next unless $DriveSupport{$pkg}{$model}; # There may be multiple target versions for each model my @targets = keys %{$DriveSupport{$pkg}{$model}}; # Check to see if the firmware is already at a target version my @results = grep {$_ eq $firmware} @targets; last if @results; # Skip this package if newer than requested next if compareVersions($pkg, $RPS{'DSP'}) > 0; foreach my $target (@targets) { # Check for firmware updates # Check if the current version is supported my @results = grep {$_ eq $firmware} @{$DriveSupport{$pkg}{$model}{$target}}; if (@results) { # Firmware updates available $dsp_target = $target; last; } } } ### # Check for nodes that support DSP foreach my $lnn (keys %{$fw->{'Counts'}}) { # Only check on Bootflash nodes next unless $NODE_INFO{$lnn}{'Bootflash'}; # Add to the list of nodes push(@{$fw->{'DSP'}{'Nodes'}}, $lnn); # Add to the count of devices $fw->{'DSP'}{'Count'} += $fw->{'Counts'}{$lnn}; # Flag FRU Package if any CTO nodes require updating if ($dsp_target and $NODE_INFO{$lnn}{'CTO'}) { $RPS{'Drive Support Package'}{'FRU'} = 1; } } # Next firmware if no updates were found next unless $dsp_target and $fw->{'DSP'}{'Nodes'} and $fw->{'DSP'}{'Count'}; ### # Beyond this point drive firmware updates are available. debug("checkDriveSupportPackage: DSP $RPS{'DSP'} supports ". "updating from '$firmware' to '$dsp_target'"); # Set target firmware version $fw->{'DSP'}{'Target'} = $dsp_target; # Add to the list of nodes to be updated @{$RPS{'Drive Support Package'}{'Nodes'}} = merge(@{$RPS{'Drive Support Package'}{'Nodes'}}, @{$fw->{'DSP'}{'Nodes'}}); # Add to the count of drives the drives to be updated $RPS{'Drive Support Package'}{'Drives'} += $fw->{'DSP'}{'Count'}; } } # Print out drive firmware information details(sprintf("%-30s %-10s %-10s %-5s %s", "Model", "Firmware", "DSP ($RPS{'DSP'})", "Count", "Nodes")); foreach my $model (keys %{$CLUSTER_INFO{'Drive Firmware'}}) { foreach my $firmware (keys %{$CLUSTER_INFO{'Drive Firmware'}{$model}}) { my $fw = $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; next unless ($fw->{'DSP'}{'Nodes'} and $fw->{'DSP'}{'Count'}); details(sprintf("%-30s %-10s %-10s %-5s %s", $model, $firmware, ($fw->{'DSP'}{'Target'} or "-"), $fw->{'DSP'}{'Count'}, compressRange(@{$fw->{'DSP'}{'Nodes'}}))) } } if (!$RPS{'Drive Support Package'}{'Nodes'}) { # Drive firmware updates not available printResults(); return; } $RPS{'Drive Support Package'}{'Target'} = "Drive Support Package $RPS{'DSP'}"; if ($RPS{'Drive Support Package'}{'FRU'}) { # Flag if the FRU package is required info("IsiFru Package is required"); } info(scalar(@{$RPS{'Drive Support Package'}{'Nodes'}}). " nodes have out of date drive firmware: ". compressRange(@{$RPS{'Drive Support Package'}{'Nodes'}})); # Flag an issue that requires nodes to be rebooted following firmware updates if ($RPS{'Drive Firmware'}{'KB490849'} and (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") <= 0 or isCurrentOneFS(qr/^(8\.0\.1\.[01])$/))) { critical(scalar(@{$RPS{'Drive Firmware'}{'KB490849'}}). " nodes are affected by KB490849: ". compressRange(@{$RPS{'Drive Firmware'}{'KB490849'}}). ". In versions OneFS 8.0.1.0, OneFS 8.0.1.1, or OneFS 8.0.0.4 and older, whenever nodes with affected Seagate drive types are updated using DSP 1.13 and newer, they must be rebooted after being updated. This can be done standalone or by any activity that reboots the nodes."); } # Flag reasons why drive firmware updates should be a priority if ($RPS{'ETA'}{'193819'}{'Nodes'}) { my @nodes = (); foreach my $lnn (@{$RPS{'ETA'}{'193819'}{'Nodes'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Support Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). " nodes are affected by ETA 193819: ". compressRange(@nodes); warning($message); push(@{$RPS{'Drive Support Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } if ($RPS{'Drive Firmware'}{'KB88930'}) { my @nodes = (); foreach my $lnn (@{$RPS{'Drive Firmware'}{'KB88930'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Support Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). " nodes are affected by KB88930: ". compressRange(@nodes); warning($message); push(@{$RPS{'Drive Support Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } if ($RPS{'Drive Firmware'}{'KB502084'}) { my @nodes = (); foreach my $lnn (@{$RPS{'Drive Firmware'}{'KB502084'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Support Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). " nodes are affected by KB502084: ". compressRange(@nodes); warning($message); push(@{$RPS{'Drive Support Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } # Remove Drive Firmware Updates if requested only priority updates and there are no priority updates needed. if ($RPS{'Priority DF Only'} and $RPS{'Drive Support Package'}{'Nodes'} and !$RPS{'Drive Support Package'}{'Priority'}) { delete $RPS{'Drive Support Package'}; printResults(); return; } # Define support for DSP on current or target version my $supports_dsp = (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") >= 0)?1:0; my $supports_dsp_ndfu = ($supports_dsp and $CLUSTER_INFO{'OneFS'} !~ /^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)?1:0; my $supports_dsp_sim = ($supports_dsp and !$supports_dsp_ndfu)?1:0; my $target_supports_dsp = ($RPS{'Target'} and compareVersions($RPS{'Target'},"7.1.1") >= 0)?1:0; my $target_supports_dsp_ndfu = ($target_supports_dsp and $RPS{'Target'} !~ /^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)?1:0; my $target_supports_dsp_sim = ($target_supports_dsp and !$target_supports_dsp_ndfu)?1:0; # Select which position to perform drive firmware updates if ($supports_dsp_ndfu) { $RPS{'Drive Support Package'}{'Position'} = 1; } elsif ($target_supports_dsp_ndfu) { $RPS{'Drive Support Package'}{'Position'} = 3; } elsif ($supports_dsp_sim) { $RPS{'Drive Support Package'}{'Position'} = 2; $RPS{'Drive Support Package'}{'Simultaneous'} = 1; addCustomerInformation("Upgrade Plan", info("Drive firmware updates using Drive Support Package $RPS{'DSP'} when performed on OneFS $CLUSTER_INFO{'OneFS'} must be performed in the DSP/Simultaneous manner, rebooting nodes one at a time but updating all drives in that node in a simultaneous manner")); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "Drive firmware updates should be completed using DSP/Simultaneous method when performed on OneFS $CLUSTER_INFO{'OneFS'}"); } elsif ($target_supports_dsp_sim) { $RPS{'Drive Support Package'}{'Position'} = 3; $RPS{'Drive Support Package'}{'Simultaneous'} = 1; addCustomerInformation("Upgrade Plan", info("Drive firmware updates using Drive Support Package $RPS{'DSP'} when performed on OneFS $RPS{'Target'} must be performed in the DSP/Simultaneous manner, rebooting nodes one at a time but updating all drives in that node in a simultaneous manner")); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "Drive firmware updates should be completed using DSP/Simultaneous method when performed on OneFS $RPS{'Target'}"); } else { addCustomerInformation("Upgrade Plan", info("Drive Support Package is not supported for drive firmware updates on OneFS $CLUSTER_INFO{'OneFS'}. Consider performing a OneFS upgrade to a version which supports the latest Drive Support Package.")); delete $RPS{'Drive Support Package'}; } # Accomodate for reboots required after using DSP and impacted by KB490849 if ($RPS{'Drive Support Package'} and $RPS{'Drive Firmware'}{'KB490849'} and !$RPS{'Drive Support Package'}{'Simultaneous'} and (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") <= 0 or $CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.1\.[01])$/)) { # Must have drives identified as affected, be running an affected OneFS version, and not performing DSP Simultaneous if (!$RPS{'Target'}) { # Not upgrading so definately affected addCustomerInformation("Upgrade Plan", "Due to issues outlined in KB490849, nodes must be rebooted after performing drive firmware updates using the Drive Support Package."); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "The subsequent reboot activity must be performed in the same maintenance window as the drive firmware update (KB490849)"); if (!$RPS{'Proactive Reboot'}) { $RPS{'Drive Firmware Reboot'}{'Position'} = "1"; @{$RPS{'Drive Firmware Reboot'}{'Nodes'}} = @{$RPS{'Drive Firmware'}{'KB490849'}}; @{$RPS{'Drive Firmware Reboot'}{'Additional Comments'}} = ("Only the following nodes require rebooting: ".compressRange(@{$RPS{'Drive Firmware'}{'KB490849'}})); } } elsif ($RPS{'Drive Support Package'}{'Position'} == 3 and $target_supports_dsp_ndfu and (compareVersions($RPS{'Target'}, "8.0.0.4") <= 0 or $RPS{'Target'} =~ /^8\.0\.1\.[01]$/)) { # Upgrading to a version that is affected addCustomerInformation("Upgrade Plan", "Due to issues outlined in KB490849, nodes must be rebooted after performing drive firmware updates using the Drive Support Package."); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "The subsequent reboot activity must be performed in the same maintenance window as the drive firmware update (KB490849)"); $RPS{'Drive Firmware Reboot'}{'Position'} = "3"; @{$RPS{'Drive Firmware Reboot'}{'Nodes'}} = @{$RPS{'Drive Firmware'}{'KB490849'}}; @{$RPS{'Drive Firmware Reboot'}{'Additional Comments'}} = ("Only the following nodes require rebooting: ".compressRange(@{$RPS{'Drive Firmware'}{'KB490849'}})); } elsif ($RPS{'Drive Support Package'}{'Position'} < 3) { # Upgrading and running drive firmware updates on affected version prior to the OneFS upgrade addCustomerInformation("Upgrade Plan", "Due to issues outlined in KB490849, nodes must be rebooted after performing drive firmware updates using the Drive Support Package."); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "The subsequent reboot or OneFS activity must be performed in the same maintenance window as the drive firmware update (KB490849)"); } else { } } # Append files required for the updates push(@{$RPS{'Drive Support Package'}{'Files'}}, "Isilon $RPS{'Drive Support Package'}{'Target'}") if $RPS{'Drive Support Package'}{'Target'}; push(@{$RPS{'Drive Support Package'}{'Files'}}, "IsiFru Package") if $RPS{'Drive Support Package'}{'FRU'}; info("Refer to KB210512 (https://support.emc.com/kb/210512) for details.") if (@INFO or @WARN or @FAIL); printResults(); } # End checkDriveFirmware $CHECKS{'checkFCOF042415EE'}{'Description'} = "Checks the cluster to see if it meets criteria for FCO F042415EE"; sub checkFCOF042415EE { return if wasCalled(); printCheck("FCO F042415EE"); gatherDriveFirmware(); if ($RPS{'FCO'}{'F042415EE'}{'Nodes'}) { # Cluster is impacted by FCO F042415EE warning("The following devices are affected by F042415EE: ". join(",", @{$RPS{'Drive Firmware'}{'F042415EE'}})); @{$RPS{'FCO F042415EE'}{'Nodes'}} = @{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}; } printResults(); } # End checkFCOF042415EE $CHECKS{'checkKB469133'}{'Description'} = "Checks for criteria of KB 469133."; sub checkKB469133 { return if wasCalled(); printCheck("FCO F031617FC/KB469133"); gatherDriveFirmware(); if ($RPS{'Drive Firmware'}{'KB469133'}) { warning(scalar(@{$RPS{'Drive Firmware'}{'KB469133'}})." drives are affected by FCO F031617FC/KB469133: ".join(", ", @{$RPS{'Drive Firmware'}{'KB469133'}})); warning("These affected drives should be smartfailed and replaced. This issue will not impact an upgrade."); } printResults(); } # End checkKB469133 ### # gatherDriveFirmware: Gathers drive firmware information from each # node and stores it in $CLUSTER_INFO{'Drive Firmware'} sub gatherDriveFirmware { return if wasCalled(); my %gather; if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi_radish -q]); } else { %gather = gatherCluster(qq[2>&1 grep -H ^Bay */isi_radish]); } # Analyze for out of date firmware my %fw = (); foreach my $lnn (@LNNS) { if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { # Gather drive information my ($bay, $model, $firmware) = ""; $bay = $1 if $line =~ /Bay\s+(\S+)\//; $model = $1 if $line =~ /is\s+(\S.*)\s+FW:/; $firmware = $1 if $line =~ /FW:(\S+)\s+SN:/; next unless $bay and $model and $firmware; # Compile %fw to contain: # Model -> Firmware -> Count/Nodes $fw{$model}{$firmware}{'Counts'}{$lnn}++; $fw{$model}{$firmware}{'Count'}++; @{$fw{$model}{$firmware}{'Nodes'}} = merge(@{$fw{$model}{$firmware}{'Nodes'}}, $lnn); # Firmware impacted by KB 502084 if (($model =~ /HITACHI HUSMM151CLAR1600/ and $firmware =~ /D344/) or ($model =~ /HGST HUSMM1680ASS205/ and $firmware =~ /D252|d322|D326/)) { @{$RPS{'Drive Firmware'}{'KB502084'}} = merge(@{$RPS{'Drive Firmware'}{'KB502084'}}, $lnn); } # Firmware impacted by KB 490849 if ($model =~ /ST.000NM00.3/ and $firmware =~ /^SN0[23]$|^SNG4$|^000[34]$/) { @{$RPS{'Drive Firmware'}{'KB490849'}} = merge(@{$RPS{'Drive Firmware'}{'KB490849'}}, $lnn); } # Firmware impacted by KB 469133 if ($firmware =~ /^SN0[46]$/) { push(@{$RPS{'Drive Firmware'}{'KB469133'}}, "$lnn:bay$bay"); } # Firmware impacted by KB 88930 if ($model =~ /HUA723030ALA640/ and $firmware =~ /MKAOA[35][8BV]0/) { @{$RPS{'Drive Firmware'}{'KB88930'}} = merge(@{$RPS{'Drive Firmware'}{'KB88930'}}, $lnn); } # Firmware impacted by FCO F042415EE if ($model =~ /HUSMM8080ASS205/ and $firmware eq "D210") { push(@{$RPS{'Drive Firmware'}{'F042415EE'}}, "$lnn:bay$bay"); @{$RPS{'FCO'}{'F042415EE'}{'Nodes'}} = merge(@{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}, $lnn); } # Firmware impacted by ETA 193819 if ($model =~ /HUS7240[234]0ALA640/ and $firmware =~ /A[8AB][BCW]0/) { @{$RPS{'ETA'}{'193819'}{'Nodes'}} = merge(@{$RPS{'ETA'}{'193819'}{'Nodes'}}, $lnn); } } } %{$CLUSTER_INFO{'Drive Firmware'}} = %fw if %fw; # Add FCO/ETA messages if ($RPS{'FCO'}{'F042415EE'}{'Nodes'}) { $RPS{'FCO'}{'F042415EE'}{'Severity'} = "WARN"; $RPS{'FCO'}{'F042415EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}). " nodes are affected by FCO F042415EE: ". compressRange(@{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}); } if ($RPS{'ETA'}{'193819'}{'Nodes'}) { $RPS{'ETA'}{'193819'}{'Severity'} = "INFO"; $RPS{'ETA'}{'193819'}{'Message'} = scalar(@{$RPS{'ETA'}{'193819'}{'Nodes'}}). " nodes are affected by ETA 193819: ". compressRange(@{$RPS{'ETA'}{'193819'}{'Nodes'}}); } } # End gatherDriveFirmware ### # checkETAs: Checks for ETAs, version specific ETAs. $CHECKS{'checkETAs'}{'Description'} = "Checks for ETAs"; sub checkETAs { return if wasCalled(); checkNodeFirmware(); printCheck("ETAs"); gatherDriveFirmware(); gatherDTA526050(); gatherDTA525928(); gatherETA519384(); gatherETA518924(); gatherETA491747(); gatherETA483840(); gatherETA209918(); gatherETA207785(); gatherETA205085(); gatherETA202452(); gatherETA198924(); gatherETA194692(); gatherETA180317(); # ETA 208773 if (isCurrentOneFS(qr/^7\.1\.1\.[67]$|^7\.2\.0\.[34]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'208773'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208773'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 208773"); } if (isDestinationOneFS(qr/^7\.1\.1\.[67]$|^7\.2\.0\.[34]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'208773'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208773'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 208773"); } # ETA 208772 if (isCurrentOneFS(qr/^7\.2\.1\.0$|^7\.2\.0\.[01234]$|^7\.1\.1\.[01234567]$|^7\.1\.0\.[0123456]$/)) { $RPS{'ETA'}{'208772'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208772'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 208772"); } if (isDestinationOneFS(qr/^7\.2\.1\.0$|^7\.2\.0\.[01234]$|^7\.1\.1\.[01234567]$|^7\.1\.0\.[0123456]$/)) { $RPS{'ETA'}{'208772'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208772'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 208772"); } # ETA 207782 if (isCurrentOneFS(qr/^7\.2\.0\.[012]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'207782'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207782'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 207782"); } if (isDestinationOneFS(qr/^7\.2\.0\.[012]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'207782'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207782'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 207782"); } # ETA 204898 if (isCurrentOneFS(qr/^7\.2\.0\.[0123]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'204898'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'204898'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 204898"); } if (isDestinationOneFS(qr/^7\.2\.0\.[0123]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'204898'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'204898'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 204898"); } # ETA 203815 if (isCurrentOneFS(qr/^7\.1\.1\.[012345]$|^7\.2\.0\.[0123]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'203815'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'203815'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 203815"); } if (isDestinationOneFS(qr/^7\.1\.1\.[012345]$|^7\.2\.0\.[012]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'203815'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'203815'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 203815"); } # ETA 200097 if (isCurrentOneFS(qr/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'200097'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200097'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 200097"); } if (isDestinationOneFS(qr/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'200097'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200097'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 200097"); } # ETA 200096 if (isCurrentOneFS(qr/^7\.1\.1\.[345]$|^7\.2\.0\.[12]$|^7\.2\.1\.1$/)) { $RPS{'ETA'}{'200096'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200096'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 200096"); } if (isDestinationOneFS(qr/^7\.1\.1\.[345]$|^7\.2\.0\.[12]$|^7\.2\.1\.1$/)) { $RPS{'ETA'}{'200096'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200096'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 200096"); } # ETA 199379 if (isCurrentOneFS(qr/^6|^7\.0|^7\.1\.0|^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'199379'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'199379'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 199379"); } if (isDestinationOneFS(qr/^6|^7\.0|^7\.1\.0|^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'199379'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'199379'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 199379"); } # ETA 198187 if (isCurrentOneFS('7.2.0.0')) { $RPS{'ETA'}{'198187'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198187'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 198187"); } if (isDestinationOneFS('7.2.0.0')) { $RPS{'ETA'}{'198187'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198187'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 198187"); } # ETA 197460 if (isCurrentOneFS('7.2.0.0')) { $RPS{'ETA'}{'197460'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'197460'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 197460"); } if (isDestinationOneFS('7.2.0.0')) { $RPS{'ETA'}{'197460'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'197460'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 197460"); } # ETA 190094 if (isCurrentOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'190094'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'190094'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 190094"); } if (isDestinationOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'190094'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'190094'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 190094"); } # ETA 189234 if (isCurrentOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'189234'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'189234'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 189234"); } if (isDestinationOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'189234'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'189234'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 189234"); } # ETA 188835 if (isCurrentOneFS('7.1.0.3')) { $RPS{'ETA'}{'188835'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'188835'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 188835"); } if (isDestinationOneFS('7.1.0.3')) { $RPS{'ETA'}{'188835'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'188835'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 188835"); } # ETA 185153 if (isCurrentOneFS(qr/^6\.5\.5\.[01][0123456789]|^6\.5\.5\.2[01]$|^7\.0\.0\.0$|^7\.0\.1\.[012345]$/)) { $RPS{'ETA'}{'185153'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'185153'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 185153"); } if (isDestinationOneFS(qr/^6\.5\.5\.[01][0123456789]|^6\.5\.5\.2[01]$|^7\.0\.0\.0$|^7\.0\.1\.[012345]$/)) { $RPS{'ETA'}{'185153'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'185153'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 185153"); } if ($RPS{'ETA'}) { foreach my $eta (sort {$a <=> $b} keys %{$RPS{'ETA'}}) { next unless $RPS{'ETA'}{$eta}{'Message'} and $RPS{'ETA'}{$eta}{'Severity'}; info(@{$RPS{'ETA'}{$eta}{'Message'}}) if $RPS{'ETA'}{$eta}{'Severity'} eq "INFO"; warning(@{$RPS{'ETA'}{$eta}{'Message'}}) if $RPS{'ETA'}{$eta}{'Severity'} eq "WARN"; fail(@{$RPS{'ETA'}{$eta}{'Message'}}) if $RPS{'ETA'}{$eta}{'Severity'} eq "FAIL"; } } printResults(); } # End checkETAs ### # gatherDTA525928: Checks to see if the cluster meets criteria for DTA 525928. sub gatherDTA525928 { return if wasCalled(); if (isCurrentOneFS(qr/^(8.0.0.7|8.1.0.4)$/)) { $RPS{'ETA'}{'525928'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'525928'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by DTA 525928."); } if (isDestinationOneFS(qr/^(8.0.0.7|8.1.0.4)$/)) { $RPS{'ETA'}{'525928'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'525928'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by DTA 525928."); } } # End gatherDTA525928 ### # gatherDTA526050: Checks to see if the cluster meets criteria for ETA 526050. sub gatherDTA526050 { return if wasCalled(); if (isCurrentOneFS(qr/^7\.[12]/)) { $RPS{'ETA'}{'526050'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'526050'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by DTA 526050"); } if (isDestinationOneFS(qr/^7\.[12]/)) { $RPS{'ETA'}{'526050'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'526050'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by DTA 526050"); } } # End gatherDTA526050 ### # gatherETA180317: Gathers whether the cluster is running versions affected by ETA 180317 sub gatherETA180317 { return if wasCalled(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'grep -m1 "QLogic 12[38]00" /var/log/opensm-[12].topo']); } else { %gather = gatherCluster(qq[2>&1 grep -m1 -H "QLogic 12[38]00" */varlog.tar/log/opensm-[12].topo]); } # Analyze information for issues. my $meets_eta180317 = 0; foreach my $lnn (@LNNS) { next unless ($gather{$lnn}); last if $meets_eta180317; foreach my $line (@{$gather{$lnn}}) { last if $meets_eta180317; $meets_eta180317 = 1 if $line =~ /QLogic 12[38]00/; } } if ($meets_eta180317) { $RPS{'ETA'}{'180317'}{'Severity'} = "WARN"; push(@{$RPS{'ETA'}{'180317'}{'Message'}}, "InfiniBand switch firmware may be running a version affected by ETA 180317"); } } # End gatherETA180317 ### # gatherETA194692: Gathers whether the cluster is running versions affected by ETA 194692 sub gatherETA194692 { return if wasCalled(); # ETA 194692 my $regex = qr/^6\.5\.5\.\d$|^6\.5\.5\.1\d$|^6\.5\.5\.2[012]$|^7\.0\.2\.[01]$|^7\.1\.0\.[012345]$|^7\.1\.1\.1$|^7\.2\.0\.0$/; if (isCurrentOneFS($regex)) { $RPS{'ETA'}{'194692'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'194692'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 194692"); } if (isDestinationOneFS($regex)) { $RPS{'ETA'}{'194692'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'194692'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 194692"); } } # End gatherETA194692 ### # gatherETA198924: Gathers whether the cluster is running versions affected by ETA 198924 sub gatherETA198924 { return if wasCalled(); gatherProductInfo(); # ETA 198924 my $has_hd400_nodes = 0; foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Product'}; $has_hd400_nodes = 1 if $NODE_INFO{$lnn}{'Product'} =~ /^HD400/; } if ($has_hd400_nodes) { if (isCurrentOneFS(qr/^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'198924'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198924'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 198924"); } if (isDestinationOneFS(qr/^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'198924'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198924'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 198924"); } } } # End gatherETA198924 ### # gatherETA202452: Gathers whether the cluster is running versions affected by ETA 202452 sub gatherETA202452 { return if wasCalled(); # ETA 202452 my $regex = qr/^7\.1\.0\.[0123456]$|^7\.1\.1\.[0123456]$|^7\.2\.0\.[0123]$|^7\.2\.1\.0$/; if (isCurrentOneFS($regex)) { $RPS{'ETA'}{'202452'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'202452'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 202452"); } if (isDestinationOneFS($regex)) { $RPS{'ETA'}{'202452'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'202452'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 202452"); } } # End gatherETA202452 ### # gatherETA205085: checks for conditions for ETA 205085. # ETA 205085: Isilon OneFS: Legacy NFSv3 clients that require 32-bit # cookies may encounter data unavailability in OneFS 7.2.1.0 and 7.2.0.0 - 7.2.0.2 sub gatherETA205085 { return if wasCalled(); return unless (compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") < 0 and $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.2") >= 0); # ETA 205085 $RPS{'ETA'}{'205085'}{'Severity'} = "WARN"; push(@{$RPS{'ETA'}{'205085'}{'Message'}}, "Upgrade from OneFS $CLUSTER_INFO{'OneFS'} to OneFS $RPS{'Target'} affected by ETA 205085"); addCustomerInformation ("ETA 205085", "NFSv3 clients that require 32-bit cookies may be unable to access data or view files that are contained in an NFS export due to a change in the way OneFS returns read directory (READDIR) cookies. This situation may result in data unavailability.", "If you have NFSv3 clients that require 32-bit cookies, review ETA 205085 (https://support.emc.com/kb/205085) and take any appropriate action prior to the upgrade."); } # End gatherETA205085 ### # gatherETA207785: Gathers whether the cluster is running versions affected by ETA 207785 sub gatherETA207785 { return if wasCalled(); # ETA 207785 if (isCurrentOneFS(qr/^7\.1\.1\.6$/)) { $RPS{'ETA'}{'207785'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207785'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 207785"); } if (isDestinationOneFS(qr/^7\.1\.1\.6$/)) { $RPS{'ETA'}{'207785'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207785'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 207785"); } } # End gatherETA207785 ### # gatherETA209918: Gathers whether the cluster is running versions affected by ETA 209918 sub gatherETA209918 { return if wasCalled(); # ETA 209918 my $regex = qr/^7\.1\.0\.[0123456]$|^7\.1\.1\.[012345]$|^7\.2\.0\.[012]$|^7\.2\.1\.0$/; if (isCurrentOneFS($regex)) { $RPS{'ETA'}{'209918'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'209918'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 209918"); } if (isDestinationOneFS($regex)) { $RPS{'ETA'}{'209918'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'209918'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 209918"); } } # End gatherETA209918 ### # gatherETA483840: checks for conditions for ETA 483840. # ETA 483840: Isilon OneFS: Isilon OneFS: Non-UTF-8 encoded files or # directories may be inaccessible to NFSv3 and NFSv4 clients after # upgrading OneFS clusters from version 7.1.x or earlier to 7.2.0.0 or later sub gatherETA483840 { return if wasCalled(); # Must be for a OneFS upgrade return unless $RPS{'Target'}; # Does not apply if already at OneFS 7.2 or higher return if compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") >= 0; # Does not apply if upgrade is not to OneFS 7.2 or higher return if compareVersions($RPS{'Target'}, "7.2") < 0; # Does not apply if upgrade is to OneFS 8.1.1 or higher return if compareVersions($RPS{'Target'}, "8.1.1") >= 0; # ETA 483840 $RPS{'ETA'}{'483840'}{'Severity'} = "WARN"; push(@{$RPS{'ETA'}{'483840'}{'Message'}}, "Upgrade from OneFS $CLUSTER_INFO{'OneFS'} to OneFS $RPS{'Target'} affected by ETA 483840"); addCustomerInformation("ETA 483840", "OneFS may inaccurately translate non-UTF-8 encoded file and directory names that are accessed by NFS clients on a small percentage of clusters that have been upgraded from OneFS 7.1.x or earlier to OneFS 7.2.0.0 or later.", "If you have NFS clients, review ETA 483840 (https://support.emc.com/kb/483840) and complete steps in the resolution to determine if this may put access to your data at risk."); } # End gatherETA483840 ### # gatherETA491747: Checks to see if the cluster meets criteria for ETA 491747. sub gatherETA491747 { return if wasCalled(); return if compareVersions($CLUSTER_INFO{'OneFS'}, "7.2.1.4") >= 0 or isCurrentOneFS("7.1.1.11"); gatherProductInfo(); gatherNodeFirmware(); my @ETA491747 = (); foreach my $lnn (@LNNS) { if ($NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(A100|X410|X210|S210|HD400|NL410)/ and $NODE_INFO{$lnn}{'Firmware'}{'Mellanox'}) { push(@ETA491747, $lnn); } } if (@ETA491747) { if (isCurrentOneFS(qr/^7\.2\.1\.[0123]$/) || isCurrentOneFS(qr/^7\.1\.1\.([0123456789]|10)$/)) { $RPS{'ETA'}{'491747'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'491747'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 491747 on ".scalar(@ETA491747)." nodes: ".compressRange(@ETA491747)); } if (isDestinationOneFS(qr/^7\.2\.1\.[0123]$/) || isDestinationOneFS(qr/^7\.1\.1\.([0123456789]|10)$/)) { $RPS{'ETA'}{'491747'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'491747'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 491747 on ".scalar(@ETA491747)." nodes: ".compressRange(@ETA491747)); } } } # End gatherETA491747 ### # gatherETA518924: Checks to see if the cluster meets criteria for ETA 518924. sub gatherETA518924 { return if wasCalled(); return if compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.1") >= 0; $RPS{'ETA'}{'518924'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'518924'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 518924"); if ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.1.1") < 0) { $RPS{'ETA'}{'518924'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'518924'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 518924"); } # A list of the last 30 days in yyyy-mm-dd format my @date = (); for (my $i = 0; $i <= 30; $i++) { my ($day, $month, $year) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4,5]; $year += 1900; $month += 1; push(@date, sprintf("%04d-%02d-%02d", $year, $month, $day)); } # A list of the error types to gather my @detect = ("bam_mark_blocks", "dfm_mark_directory", "ifm_mark_metatree", "qdb_mark_inner", "sbt_mark_tree", "stf_mark_metatree"); # Compile an awk command my $awkstring = "BEGIN{IGNORECASE=1}"; foreach my $detect_string (@detect) { my $detect_key = $detect_string; $detect_key =~ s/[^a-zA-Z0-9]/_/g; $awkstring .= qq[/$CLUSTER_INFO{'Name'}.*${detect_string}/{${detect_key}++; ${detect_key}MR=\\\$1}]; $awkstring .= qq[END{if(${detect_key}){print \\\"${detect_key}:\\\" ${detect_key} \\\" MR:\\\" ${detect_key}MR}}]; } # Compile greps from the lists my $date_grep = "^(".join("|", @date).")"; my $detect_grep = join("|", @detect); my $ignore_grep = "sudo: "; # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array ']. qq[awk "$awkstring"]. qq[ <(grep -E "$detect_grep" /var/log/messages]. qq[|grep -E "$date_grep"]. qq[|grep -E -v "$ignore_grep")']); } else { $awkstring =~ s/\\//g; %gather = gatherCluster(qq[for i in */varlog.tar/log/messages; do 2>&1]. qq[ grep -H -E "$date_grep" \$i]. qq[|grep -E "$detect_grep"]. qq[|grep -E -v "$ignore_grep"]. qq[|awk '$awkstring']. qq[|awk 'BEGIN{FS=":";OFS=":"}{LNN=\$3;\$3=\$2;\$2=\$1;\$1=LNN;print}']. qq[;done]); } # Analyze information for issues. my $most_recent_timestamp = ""; my $most_recent_lnn = ""; foreach my $lnn (@LNNS) { foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /MR:(.*)$/; if ($1 gt $most_recent_timestamp) { $most_recent_timestamp = $1; $most_recent_lnn = $lnn; } } } warning("ETA 518924: Locking contention seen in the last 30 days, most recent occurrence seen on node $most_recent_lnn at $most_recent_timestamp") if $most_recent_timestamp; } # End gatherETA518924 sub gatherETA519384 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.6") < 0 or isCurrentOneFS(qr/^(8\.0\.1\.[01]|8\.1\.0\.0)$/)) { $RPS{'ETA'}{'519384'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'519384'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 519384"); } if ($RPS{'Target'} and (compareVersions($RPS{'Target'}, "8.0.0.6") < 0 or isDestinationOneFS(qr/^(8\.0\.1\.[01]|8\.1\.0\.0)$/))) { $RPS{'ETA'}{'519384'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'519384'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 519384"); } } # End gatherETA519384 $CHECKS{'checkAggregationMode'}{'Description'} = "Checks that the aggregation mode is not Legacy FEC mode for upgrades to OneFS 8+"; sub checkAggregationMode { return if wasCalled(); return unless ($RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0 and compareVersions($RPS{'Target'}, "8.0") >= 0); gatherNetworkPools(); return unless $CLUSTER_INFO{'Network Pools'}; printCheck("Legacy Aggregation Mode"); debug("checkAggregationMode: Checking for network pools configured with Legacy FEC Aggregation Modes"); my @network_pools_with_legacy_fec = (); foreach my $name (keys %{$CLUSTER_INFO{'Network Pools'}}) { # Check for Legacy FEC mode for the network pool next unless $CLUSTER_INFO{'Network Pools'}{$name}{'Aggregation Mode'} and $CLUSTER_INFO{'Network Pools'}{$name}{'Aggregation Mode'} =~ /Legacy FEC/; # Check that the pool has any aggregate network interfaces next unless $CLUSTER_INFO{'Network Pools'}{$name}{'Ifaces'} and grep(/agg/, @{$CLUSTER_INFO{'Network Pools'}{$name}{'Ifaces'}}); # Both conditions met, flag as impacted debug("checkAggregationMode: !!! Network Pool $name uses Aggregation Mode $CLUSTER_INFO{'Network Pools'}{$name}{'Aggregation Mode'}"); push(@network_pools_with_legacy_fec, $name); } if (@network_pools_with_legacy_fec) { warning("Legacy link aggregation in use on the following subnet/pools: ".join(", ", @network_pools_with_legacy_fec)); addCustomerInformation("Your cluster has at least one pool with \"legacy\" link aggregation in use. This mode is no longer supported starting in OneFS v8.0.0. As part of the upgrade the affected pool(s) will be switched to a compatible FEC mode.interface Please refer to the OneFS Upgrade Planning and Process Guide at support.emc.com. Affected subnets / pools: ".join(", ", @network_pools_with_legacy_fec)); } info("Refer to KB486561 (https://support.emc.com/kb/486561) for details.") if (@WARN or @FAIL); printResults(); } # End checkAggregationMode ### # checkBMC: checks for BMC related issues $CHECKS{'checkBMCandCMC'}{'Description'} = "Checks for BMC/CMC related issues"; sub checkBMCandCMC { return if wasCalled(); checkHardwareStatus(); printCheck("BMC/CMC Hardware Monitoring"); gatherBMCInfo(); gatherCMCInfo(); gatherProductInfo(); gatherNodeFirmware(); # Look for BMC/CMC issues my @node_list = (); my %devices_reporting_none = (); foreach my $lnn (@LNNS) { # Only affects S210, X210, X410, NL410 or HD400 nodes. next unless ($NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(S210|X210|X410|HD400|NL410)/); # If the CMC firmware is not up to 02.05 or unresponsive if (!$NODE_INFO{$lnn}{'Firmware'}{'CMC'} or compareVersions($NODE_INFO{$lnn}{'Firmware'}{'CMC'}, "02.05") < 0) { push(@{$RPS{'CMC Firmware'}{'Nodes'}}, $lnn); } # If the BMC firmware is not up to 01.25 or unresponsive if (!$NODE_INFO{$lnn}{'Firmware'}{'BMC'} or compareVersions($NODE_INFO{$lnn}{'Firmware'}{'BMC'}, "01.25") < 0) { push(@{$RPS{'BMC Firmware'}{'Nodes'}}, $lnn); } # Check for hardware monitoring issues my $hardware_monitoring_issue = 0; # If BMC or CMC is not found/unresponsive unless (not $RPS{'Live'} and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") == 0) { if (!$NODE_INFO{$lnn}{'Firmware'}{'BMC'}) { debug("checkBMCandCMC: !!! Node $lnn BMC not responsive !!!"); $hardware_monitoring_issue = 1; } if (!$NODE_INFO{$lnn}{'Firmware'}{'CMC'}) { debug("checkBMCandCMC: !!! Node $lnn CMC not responsive !!!"); $hardware_monitoring_issue = 1; } } # If both power supplies don't report FW versions if (!$NODE_INFO{$lnn}{'Firmware'}{'PS1'} and !$NODE_INFO{$lnn}{'Firmware'}{'PS2'}) { debug("checkBMCandCMC: !!! Node $lnn Power supplies not reporting !!!"); $hardware_monitoring_issue = 1; } # If any ^(BMC/CAR/CFFPS/CMCSDR/CMC/CMD) component reports None, flag as BMC/CMC issues my @results = grep {$_ =~ /^(BMC|CAR|CMCSDR|CMC|CMD)/} keys %{$NODE_INFO{$lnn}{'Firmware'}}; foreach my $device (@results) { if ($NODE_INFO{$lnn}{'Firmware'}{$device} eq "None") { $hardware_monitoring_issue = 1; push(@{$devices_reporting_none{$device}}, $lnn); debug("checkBMCandCMC: !!! Node $lnn Device $device is reporting None"); } } # Add to list of nodes with hardware monitoring issues if ($hardware_monitoring_issue) { debug("checkBMCandCMC: Found a hardware monitoring issue on node $lnn!"); push(@node_list, $lnn); } } if ($RPS{'BMC Firmware'}{'Nodes'} or $RPS{'CMC Firmware'}{'Nodes'}) { # Flag that BMC/CMC is assumed out of date if not responsive/gathered info("Be advised, nodes with unresponsive BMC/CMC or not reporting a BMC/CMC firmware version are assumed to be out of date") if @node_list; # Flag that BMC is assumed to be out of date for log gathers in OneFS 7.1.1 info("Be advised, log gathers in OneFS 7.1.1 does not include BMC firmware versions, BMC firmware versions are assumed to be out of date") if (not $RPS{'Live'} and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") == 0); } # BMC firmware out of date on some nodes if ($RPS{'BMC Firmware'}{'Nodes'}) { # Flag BMC firmware FCO details $RPS{'FCO'}{'F010517EEB'}{'Severity'} = "INFO"; $RPS{'FCO'}{'F010517EEB'}{'Message'} = scalar(@{$RPS{'BMC Firmware'}{'Nodes'}})." nodes are running BMC firmware versions affected by FCO F010517EE: ".compressRange(@{$RPS{'BMC Firmware'}{'Nodes'}}); info($RPS{'FCO'}{'F010517EEB'}{'Message'}); # BMC Firmware Activity: $RPS{'BMC Firmware'}{'Target'} = "BMC Firmware Package 1.0"; push(@{$RPS{'BMC Firmware'}{'Files'}}, "Isilon BMC Firmware Package 1.0"); push(@{$RPS{'BMC Firmware'}{'Priority'}}, "BMC firmware updates are strongly recommended because ".scalar(@{$RPS{'BMC Firmware'}{'Nodes'}})." nodes are affected by F010517EE"); # Set the position of where the BMC firmware updates will be completed. if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.2") >= 0) { $RPS{'BMC Firmware'}{'Position'} = 1; } elsif ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.0.0.2") >= 0) { $RPS{'BMC Firmware'}{'Position'} = 2; } else { warning("BMC firmware updates cannot be performed remotely by RPS on OneFS versions prior to 8.0.0.2."); } } # CMC firmware out of date on some nodes if ($RPS{'CMC Firmware'}{'Nodes'}) { $RPS{'FCO'}{'F010517EEC'}{'Severity'} = "INFO"; $RPS{'FCO'}{'F010517EEC'}{'Message'} = scalar(@{$RPS{'CMC Firmware'}{'Nodes'}})." nodes are running CMC firmware versions affected by FCO F010517EE: ".compressRange(@{$RPS{'CMC Firmware'}{'Nodes'}}); info($RPS{'FCO'}{'F010517EEC'}{'Message'}); } # Flag CMC/BMC components/devices reporting None foreach my $device (keys %devices_reporting_none) { fail("The $device device is reporting None on the following nodes: ". compressRange(@{$devices_reporting_none{$device}})); } fail("Hardware monitoring issues detected on nodes: ".compressRange(@node_list)) if @node_list; info("Refer to KB489050 (https://support.emc.com/kb/489050) for details.") if (@WARN or @FAIL); printResults(); } # End checkBMCandCMC $CHECKS{'checkBootDisks'}{'Description'} = "Checks boot disk wear life remaining, firmware level, and historic error count. For Generation 4/5, checks for a master and slave boot device."; sub checkBootDisks { return if wasCalled(); gatherProductInfo(); return if $CLUSTER_INFO{'Virtual'}; printCheck("Boot Disks"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'isi_radish -a "Internal J3" "Internal J4" "Carrier board J3" "Carrier board J4"']); } else { %gather = gatherCluster(qq[2>&1 grep -H . */isi_radish_bootdisk]); } # Analyze information for issues. my @fw_702k = (); foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Boot Disks"); next if $NODE_INFO{$lnn}{'Accelerator'}; next if $NODE_INFO{$lnn}{'Generation'} and $NODE_INFO{$lnn}{'Generation'} eq "IQ"; if (!$gather{$lnn}) { missing($lnn); next; } my $bd_location = ""; my $bd_details = ""; my $has_master = 0; my $has_slave = 0; foreach my $line (@{$gather{$lnn}}) { if ($line =~ /^\s*(?:Internal|Carrier board)?\s*(\S+\/\S+)\s+is\s+(.*)$/i) { debug("checkBootDisks: Node $lnn: $line"); $bd_location = $1; $bd_details = $2; verbose("Boot Device $bd_location:"); if ($bd_location =~ /ada0|ad0|ad2|ad4|ad6/) { $has_master = 1; verbose(" (Master device)"); } if ($bd_location =~ /ada1|ad1|ad3|ad5|ad7/) { $has_slave = 1; verbose(" (Slave device)"); } verbose(" Model: $1") if $bd_details =~ /^(.*?) FW:/; verbose(" Serial Number: $1") if $bd_details =~ /SN:([^,]+)/; verbose(" Firmware: $1") if $bd_details =~ /FW:(.*?) SN/; push(@fw_702k, "Node $lnn $bd_location") if $bd_details =~ /FW:.*7\.02k/; } if ($line =~ /(?:Percent Life Remaining|Percent Lifetime Left).*raw (\S+)\)/i) { debug("checkBootDisks: Node $lnn: $line"); my $liferemaining = sprintf("%d", hex($1)); verbose(" Wear Life Remaining: $liferemaining\%"); if ($liferemaining <= 5) { fail("Boot disk $bd_location on node $lnn has $liferemaining\% life remaining."); } elsif ($liferemaining <= 10) { warning("Boot disk $bd_location on node $lnn has $liferemaining\% life remaining."); } } elsif ($line =~ /error index\s+\d+\s+historic error count\s+(\d+)$/i) { debug("checkBootDisks: Node $lnn: $line"); verbose(" Historic Error Count: $1"); warning("Boot disk at $bd_location on node $lnn has a historic error count of $1") if ($1 ne "0000"); } } if ($NODE_INFO{$lnn}{'Generation'} and $NODE_INFO{$lnn}{'Generation'} ne "Gen6" and not $NODE_INFO{$lnn}{'Accelerator'}) { if (not $has_master) { fail("Node $lnn master boot disk device information not found."); } if (not $has_slave) { fail("Node $lnn slave boot disk device information not found."); } } } warning("The following boot devices are running firmware version 7.02k which is impacted by KB489143: ".join(", ", @fw_702k)) if @fw_702k; info("Refer to KB210514 (https://support.emc.com/kb/210514) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkBootDisks $CHECKS{'checkBXENodes'}{'Description'} = "Checks for existence of nodes with BXE interfaces, also checks for KB493493 & KB511208 known issues."; sub checkBXENodes { # Pre-requisite version checks return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") < 0) { debug("checkBXENodes: Skipped.."); return; } # Print showing check running printCheck("BXE Nodes"); # Gather required information gatherNetworkInterfaces(); # Flag nodes with BXE interfaces or that no nodes have BXE interfaces my $num_bxe_nodes = 0; if ($CLUSTER_INFO{'BXE Nodes'}) { $num_bxe_nodes = scalar(@{$CLUSTER_INFO{'BXE Nodes'}}); details("$num_bxe_nodes nodes have BXE interfaces: ".compressRange(@{$CLUSTER_INFO{'BXE Nodes'}})); } # Flag issue in KB 493493 checkKB493493(); # Flag issue in KB 511208 checkKB511208(); # Output results printResults(); } # End checkBXENodes $CHECKS{'checkDriveLoad'}{'Description'} = "Checks the current load on the drives"; sub checkDriveLoad { return if wasCalled(); printCheck("Drive Load"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi statistics drive --degraded]); } else { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi statistics drive --degraded --long]); } } else { %gather = gatherCluster(qq[2>&1 grep -H . */isi_statistics_drive]); } # Analyze information for issues. my ($bay, $typ, $tiq, $que, $max); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { $bay = 0; $typ = 1; $tiq = 7; $que = 8; $max = 9; } else { $bay = 1; $typ = 2; $tiq = 11; $que = 12; $max = 15; } my @devices_highload = (); foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Drive Load"); verbose(sprintf("%-10s %-10s %-15s %-10s", "Location", "Type", "Time-in-queue", "Queue")); verbose(sprintf("%-10s %-10s %-15s %-10s", "-"x10, "-"x10, "-"x15, "-"x10)); if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { $line =~ s/^\s+|\s+$//g; next if $line =~ /^(LNN|Drive|Timestamp|-|Total)/; my @device = split(/\s+/, $line); next unless @device and $#device == $max; $device[$tiq] =~ s/ms//; verbose(sprintf("%-10s %-10s %-15s %-10s", $device[$bay], $device[$typ], $device[$tiq], $device[$que])); if (($device[$typ] eq 'SAS' and ($device[$tiq] >= 80 or $device[$que] >= 10)) or ($device[$typ] eq 'SATA' and ($device[$tiq] >= 100 or $device[$que] >= 5)) or ($device[$typ] eq 'SSD' and ($device[$tiq] >= 5 or $device[$que] >= 1))) { push(@devices_highload, $device[$bay]); } } } warning("High disk load detected on the following devices: ".join(",", @devices_highload)) if @devices_highload; # Output Check result printResults(); } # End checkDriveLoad $CHECKS{'checkDrivesHealth'}{'Description'} = "Checks health of drives and the drive stall timeout setting in sysctl"; sub checkDrivesHealth { return if wasCalled(); printCheck("Drives Health"); gatherProductInfo(); # Gather device information my %gather = (); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0 and $RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi devices|grep -v HEALTHY"); } elsif (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { %gather = gatherCluster("2>&1 grep -H -v HEALTHY */isi_devices"); } elsif ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array 'isi devices drive list|grep -v HEALTHY'"); } else { %gather = gatherCluster("2>&1 grep -H -v HEALTHY */isi_devices_drive_list"); } verboseHeader("Drives Health - Drives in a HEALTHY state are not reported"); # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Drives Health"); verbose(sprintf("%-10s %-10s", "Location", "State")); verbose(sprintf("%-10s %-10s", "-"x10, "-"x10)); if (!$gather{$lnn}) { missing($lnn); next; } my $unpack_template = ""; foreach my $line (@{$gather{$lnn}}) { next if ($line =~ /^[\s-]*$/); next if ($line =~ /^\s*Total/); # Strip leading whitespace $line =~ s/^\s*//; my ($bay, $lnum, $state) = ""; my @unpacked_line = (); if ($line =~ /^\s*Lnn/) { $unpack_template = generateUnpackTemplate($line, "Lnn", "Location", "Device", "Lnum", "State", "Serial"); next; } elsif ($unpack_template) { eval { @unpacked_line = unpack($unpack_template, $line); $bay = $1 if ($unpacked_line[1] =~ /Bay\s+(\S+)/); $lnum = $1 if ($unpacked_line[3] =~ /(\d+)/); $state = $1 if ($unpacked_line[4] =~ /(\S+)/); }; } else { $bay = $1 if ($line =~ /Bay\s+(\S+)/); $lnum = $1 if ($line =~ /Lnum\s+(\S+)/); $state = $1 if ($line =~ /\[(.*)\]/); } next unless ($lnum or $bay) and $state; my $identifier = ($bay?"Bay $bay":"Lnum $lnum"); verbose(sprintf("%-10s %-10s", $identifier, $state)); # Flag nodes if the SSDs are L3, this will be used for drive firmware and FCO activity scheduling if ($state =~ /L3/) { $NODE_INFO{$lnn}{'L3'} = 1; debug("NODE_INFO->${lnn}->L3=1"); } # Healthy states next if ($state =~ /HEALTHY|L3|BOOT_DRIVE|JOURNAL/); # Expected to see EMPTY drives on a diskless/accelerator node next if ($state =~ /EMPTY/ and $NODE_INFO{$lnn}{'Accelerator'}); next if ($state =~ /EMPTY/ and $CLUSTER_INFO{'Gen6'} and grep {$lnn eq $_} @{$CLUSTER_INFO{'Gen6'}}); next if ($state =~ /EMPTY/ and $CLUSTER_INFO{'Virtual'} and grep {$lnn eq $_} @{$CLUSTER_INFO{'Virtual'}}); # Not healthy, but the upgrade can proceed if ($state =~ /STALLED|REPLACE|EMPTY|ERASE|USED/) { warning("Node $lnn $identifier is $state"); } # Unexpected state else { fail("Node $lnn $identifier is $state"); } } } info("Refer to KB210511 (https://support.emc.com/kb/210511) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkDrivesHealth $CHECKS{'checkDriveStallTimeout'}{'Description'} = "Checks current Drive Stall Timeout setting, recommend value is 3.5 seconds (3500000 microseconds) or higher"; sub checkDriveStallTimeout { return if wasCalled(); printCheck("Drive Stall Timeout"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array /sbin/sysctl hw.disk_event.thresh.slowacc_usec]); } else { %gather = gatherCluster(qq[2>&1 grep -H hw.disk_event.thresh.slowacc_usec */hw.disk_event]); } # Analyze information for issues. my $min_recommended_threshold = 3500000; my @nodes_less_than_threshold = (); verboseHeader("Drive Stall Timeout"); verbose(sprintf("%-10s %-10s", "Node", "Drive Stall Timeout Setting (microseconds)")); verbose(sprintf("%-10s %-10s", "-"x10, "-"x10)); foreach my $lnn (@LNNS) { if (!$gather{$lnn}) { missing($lnn); next; } my $threshold = 0; foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /hw.disk_event.thresh.slowacc_usec: (\d+)$/; $threshold = $1; verbose(sprintf("%-10s %-10s", $lnn, $threshold)); } if ($threshold and $threshold < $min_recommended_threshold) { push(@nodes_less_than_threshold, $lnn); } } if (@nodes_less_than_threshold) { warning("Drive stall timeout is not set to 3.5 seconds on the following nodes: ".compressRange(@nodes_less_than_threshold), "For details on increasing the drive stall timeout, see KB487685: https://support.emc.com/kb/487685 - Increasing the drive stall timeout values in OneFS"); } printResults(); } # End checkDriveStallTimeout sub checkExpectedMemory { return if wasCalled(); checkHardwareStatus(); verboseHeader("checkExpectedMemory"); verbose(sprintf("%5s %10s %10s", "LNN", "Actual", "Expected")); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'RAM'}; # Current amount of RAM reported in isi_hw_status my $mem_current_int = $NODE_INFO{$lnn}{'RAM'}; my $mem_current_str = convertFromNumeric($mem_current_int, "G"); # Expected amount of RAM from the product info my @product_info = split(/-/, $NODE_INFO{$lnn}{'Product'}); # This won't work for older/IQ nodes next unless scalar(@product_info) > 3; my $mem_expected_str = $product_info[3]; my $mem_expected_int = convertToNumeric($mem_expected_str); # If the expected and current memory are more than 1GB off if (abs($mem_expected_int - $mem_current_int) > 1073741824) { fail("Memory mismatch node $lnn, expected $mem_expected_str, found $mem_current_str."); } # Verbose output verbose(sprintf("%5s %10s %10s", $lnn, $mem_current_str, $mem_expected_str)); } } # End checkExpectedMemory $CHECKS{'checkGatewayPriority'}{'Description'} = "Checks for subnets with duplicate gateway priorities."; sub checkGatewayPriority { return if wasCalled(); gatherNetworkSubnets(); return unless $CLUSTER_INFO{'Network Subnets'}; printCheck("Duplicate Gateway Priority"); debug("checkGatewayPriority: Creating hash of array for each subnet priority."); my %subnet_by_priority = (); verboseHeader("Gateway Priority"); verbose(sprintf("%-30s %-20s %-10s", "Subnet", "Gateway", "Priority")); foreach my $id (keys %{$CLUSTER_INFO{'Network Subnets'}}) { my %subnet = %{$CLUSTER_INFO{'Network Subnets'}{$id}}; next unless $subnet{'Priority'} and $subnet{'Gateway'}; next if $subnet{'Gateway'} eq "0.0.0.0"; push(@{$subnet_by_priority{$subnet{'Priority'}}}, $id); verbose(sprintf("%-30s %-20s %-10s", $id, $subnet{'Gateway'}, $subnet{'Priority'})); } debug("checkGatewayPriority: Checking for duplicate gateway priorities"); my $found_duplicate_gateway_priorities = 0; foreach my $priority (keys %subnet_by_priority) { debug("checkGatewayPriority: Gateway priority of $priority for subnets: ".join(", ", @{$subnet_by_priority{$priority}})); next unless (scalar(@{$subnet_by_priority{$priority}} > 1)); $found_duplicate_gateway_priorities = 1; warning("Duplicate gateway priority of $priority found for subnets: ".join(", ", @{$subnet_by_priority{$priority}})); debug("checkGatewayPriority: !!! Duplicate gateway priority found for gateway priority $priority !!!"); } addCustomerInformation("It was detected that multiple network subnets have the same gateway priority. This could result in unpredictable routing issues and lead to temporary data unavailability. See KB462780 (https://support.emc.com/kb/462780) for more information and the Isilon External Network Connectivity Guide (https://support.emc.com/docu58740).") if $found_duplicate_gateway_priorities; info("Refer to KB485036 (https://support.emc.com/kb/485036) for details") if (@WARN or @FAIL); printResults(); } # End checkGatewayPriority $CHECKS{'checkHardwareStatus'}{'Description'} = "Checks battery health, power supplies, and gathers hardware details for use elsewhere."; sub checkHardwareStatus { return if wasCalled(); printCheck("Hardware Status"); # Gather information my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array \"perl -e 'eval{local \\\$SIG{ALRM}=sub{kill -9,-\\\$PID};alarm 300;system(qq[${SUDO}isi_hw_status -Lcisfvtbg|grep -v test]);alarm 0;};'\""); } else { %gather = gatherCluster("2>&1 grep -H -v test */isi_hw_status"); } # Analyze information my $kb519119 = 0; foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Hardware Status"); if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { verbose($line); if ($line =~ /Write to lm96000 fan2 configuration register failed/) { fail("Node $lnn is reporting '${line}'"); } elsif ($line =~ /hal_get_power_supply_type/) { fail("Node $lnn is reporting '${line}'"); } elsif ($line =~ /^\s*Product: (.*)$/) { $NODE_INFO{$lnn}{'Product'} = $1; debug("NODE_INFO->${lnn}->Product=$1"); } elsif ($line =~ /^\s*Series: (.*)$/) { $NODE_INFO{$lnn}{'Series'} = $1; debug("NODE_INFO->${lnn}->Series=$1"); } elsif ($line =~ /^\s*PROC: (.*)$/) { $NODE_INFO{$lnn}{'Cores'} = getCoreCount($1); debug("NODE_INFO->${lnn}->Cores=$NODE_INFO{$lnn}{'Cores'}"); } elsif ($line =~ /^\s*PwrSupl: (\S+)/) { my $location = $1; my $firmware = ""; my $type = ""; $firmware = $1 if $line =~ /fw=(\S+)/; $type = $1 if $line =~ /type=(\S+)/; if ($firmware ne "N/S") { $NODE_INFO{$lnn}{'Firmware'}{$location} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${location}=$firmware"); } if ($NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(A200|A2000|H400|H500|H600|F800)/ and $type !~ /^ARTESYN/) { $kb519119 = 1; } } elsif ($line =~ /^\s*Power Supply\s+(\S+)\s+(.*)$/) { next if $line =~ /=/; if ($2 ne "good") { fail("Power Supply $1 on node $lnn is $2"); } } elsif ($line =~ /^\s*Battery1:\s*\((\S*)\)\s*Battery2:\s*\((\S*)\)\s*$/) { if ($1 ne "okay") { fail("Battery 1 on node $lnn is $1"); } if ($2 ne "okay") { fail("Battery 2 on node $lnn is $2"); } } elsif ($line =~ /^\s*RAM:\s*(\d+)\s*Bytes$/) { $NODE_INFO{$lnn}{'RAM'} = $1; debug("NODE_INFO->${lnn}->RAM=$1"); } } } fail("This cluster contains Generation 6 nodes that may be impacted by KB 519119") if ($kb519119); info("Refer to KB210507 (https://support.emc.com/kb/210507) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkHardwareStatus $CHECKS{'checkIBInterfaces'}{'Description'} = "Checks for ib0/1 as being active, checks for ETA180317 IB switch firmware versions, and checks for overlapping IB networks."; sub checkIBInterfaces { return if wasCalled(); checkProcesses(); printCheck("IB Interfaces Active"); gatherNetworkInterfaces(); gatherArrayXML(); gatherETA180317(); my @nodes_with_issues = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); next unless $NODE_INFO{$lnn}{'Network'} and $CLUSTER_INFO{'Active IB Interfaces'}; verbose("ib0 Status: $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'}") if $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'}; verbose("ib1 Status: $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'}") if $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'}; if ($CLUSTER_INFO{'Active IB Interfaces'} == 1) { if (($NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} eq 'active') or ($NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} eq 'active')) { debug("checkIBInterfaces: Node $lnn passes IB interfaces active check"); } else { push(@nodes_with_issues, $lnn); debug("checkIBInterfaces: Node $lnn does not have at least one active IB interface"); } } elsif ($CLUSTER_INFO{'Active IB Interfaces'} == 2) { if (($NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} eq 'active') and ($NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} eq 'active')) { debug("checkIBInterfaces: Node $lnn passes IB interfaces active check"); } else { push(@nodes_with_issues, $lnn); debug("checkIBInterfaces: Node $lnn does not have at least two active IB interface"); } } } # Check for overlapping IB networks if ($CLUSTER_INFO{'Interfaces'}) { debug("checkIBInterfaces: Searching for minimum netmask..."); my $min_netmask = 0b11111111111111111111111111111111; foreach my $interface (sort keys %{$CLUSTER_INFO{'Interfaces'}}) { verboseHeader("Internal Interface $interface"); verbose("Flags: $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'}; verbose("Low: $CLUSTER_INFO{'Interfaces'}{$interface}{'Low'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'Low'}; verbose("High: $CLUSTER_INFO{'Interfaces'}{$interface}{'High'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'High'}; verbose("Netmask: $CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}; verbose("MTU: $CLUSTER_INFO{'Interfaces'}{$interface}{'MTU'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'MTU'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'} =~ /enabled/; my $bin_netmask = convertIPToNumeric($CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}); $min_netmask = $bin_netmask if not defined $min_netmask or $bin_netmask < $min_netmask; } debug("checkIBInterfaces: Minimum mask: ".sprintf("%b", $min_netmask)); debug("checkIBInterfaces: Evaluating for overlap between IB networks..."); my @seen_interfaces = (); foreach my $interface_a (keys %{$CLUSTER_INFO{'Interfaces'}}) { next unless $CLUSTER_INFO{'Interfaces'}{$interface_a}{'Netmask'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_a}{'Flags'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_a}{'Flags'} =~ /enabled/; push(@seen_interfaces, $interface_a); my $a_lowaddr_bin = convertIPToNumeric($CLUSTER_INFO{'Interfaces'}{$interface_a}{'Low'}); foreach my $interface_b (keys %{$CLUSTER_INFO{'Interfaces'}}) { next if $interface_a eq $interface_b; next unless $CLUSTER_INFO{'Interfaces'}{$interface_b}{'Netmask'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_b}{'Flags'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_b}{'Flags'} =~ /enabled/; my @results = grep {$_ eq $interface_b} @seen_interfaces; next if @results; my $b_lowaddr_bin = convertIPToNumeric($CLUSTER_INFO{'Interfaces'}{$interface_b}{'Low'}); if (($a_lowaddr_bin & $min_netmask) == ($b_lowaddr_bin & $min_netmask)) { debug("checkIBInterfaces: Found overlap between $interface_a and $interface_b"); warning("Overlap found between IB interfaces $interface_a and $interface_b"); } else { debug("checkIBInterfaces: No overlap between $interface_a and $interface_b"); } } } } warning(@{$RPS{'ETA'}{'180317'}{'Message'}}) if $RPS{'ETA'}{'180317'}{'Message'}; fail("The following nodes do not have $CLUSTER_INFO{'Active IB Interfaces'} active IB interfaces: ".join(", ", @nodes_with_issues)) if @nodes_with_issues; info("Refer to KB515571 (https://support.emc.com/kb/515571) for details.") if (@WARN or @FAIL); printResults(); } # End checkIBInterfaces sub checkKB493493 { # Pre-requisite version checks return if wasCalled(); if ($CLUSTER_INFO{'OneFS'} !~ /^8\.0\.0\.[01234]$/) { debug("checkKB493493: OneFS version is not 8.0.0.[01234]. Skipped.."); return; } # Gather NIC driver if available and patches information gatherPatches(); gatherBXEDriver(); gatherNetworkInterfaces(); # Based on the code level, try to make assumptions on the BXE NIC driver my $safe_driver_codelevel = 0; if ($CLUSTER_INFO{'OneFS'} =~ /^8\.0\.0\.[012]$/) { $safe_driver_codelevel = 1; } if (hasPatch("188239")) { $safe_driver_codelevel = 1; } if (hasPatch("182721")) { $safe_driver_codelevel = 0; } # Check to see if any interfaces are impacted my %lagg_interfaces = (); foreach my $lnn (@LNNS) { # On a known impacted BXE driver version or next unless $NODE_INFO{$lnn}{'dev.bxe.0.version'} and $NODE_INFO{$lnn}{'dev.bxe.0.version'} =~ /^1\.78\.89\.ISILON\.0[234]$/; # Wasn't able to gather BXE nic information, likely a logset, # Use assumptions to predict if they will be impcated next if !$NODE_INFO{$lnn}{'dev.bxe.0.version'} and $safe_driver_codelevel; my @affected_interfaces = (); foreach my $interface (keys %{$NODE_INFO{$lnn}{'Network'}}) { push(@affected_interfaces, $interface) if ($interface =~ /lagg/i and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} =~ /lacp/i and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'} and grep {$_ =~ /^bxe/} @{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'}}); } foreach (@affected_interfaces) { push(@{$lagg_interfaces{$_}}, $lnn); } push(@{$RPS{'KB493493'}}, $lnn) if @affected_interfaces; } foreach (keys %lagg_interfaces) { warning("$_ interface affected by KB493493 on nodes: ".compressRange(@{$lagg_interfaces{$_}})); } # If identified as impacted by KB493493, patch 188239 should be performed pre-upgrade, prior to any node reboots if ($RPS{'KB493493'}) { if (hasPatch("188239")) { @{$RPS{'Proactive Reboot'}{'Nodes'}} = @{$RPS{'KB493493'}}; addCustomerInformation("Upgrade Plan", warning("Patch 188239 is installed, but the following nodes were detected as running old NIC drivers which need to be rebooted with lagg interfaces first being taken down on each node prior to reboot: ".compressRange(@{$RPS{'KB492492'}}))); } else { # Provide a reason why we are doing the patch/reboot addCustomerInformation("Upgrade Plan", "Installation of patch 188239 is required to avoid node panics caused by issues outlined in KB493493."); # This will make sure it gets into the upgrade plan $RPS{'Pre Upgrade Patches'}{'patch-188239'}{'Impacts'} = "During installation or removal of patch 188239, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete."; @{$RPS{'Pre Upgrade Patches'}{'patch-188239'}{'Reboot'}} = @{$CLUSTER_INFO{'BXE Nodes'}}; } } } # End checkKB493493 sub checkKB511208 { return if wasCalled(); return if (isCurrentOneFS(qr/^8\.0\.0/) and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.7") >= 0) or (isCurrentOneFS(qr/^8\.1\.0/) and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.4") >= 0) or compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.1.1") >= 0; gatherNetworkInterfaces(); # Analyze information for issues. my @impacted = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Network'}; my @lagg_interfaces = grep {$_ =~ /^lagg/} keys %{$NODE_INFO{$lnn}{'Network'}}; foreach my $lagg_interface (@lagg_interfaces) { next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'} =~ /lacp/; foreach my $laggport (@{$NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}}) { my $interface = ""; my $flags = ""; $interface = $1 if $laggport =~ /^(\S+)/; $flags = $1 if $laggport =~ /flags=(\S+)/; next unless $interface and $interface =~ /bxe/; next unless $flags; next unless $NODE_INFO{$lnn}{'Network'}{$interface}{'status'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'status'} eq "active"; next unless $flags =~ /0<>/; # next if $flags =~ /ACTIVE/ and $flags =~ /COLLECTING/ and $flags =~ /DISTRIBUTING/; push(@impacted, "Node $lnn, aggregate interface $lagg_interface is reporting 'flags=0<>' for the interface $interface"); } } } # Skip unless identified as impacted return unless @impacted; if ((isDestinationOneFS(qr/^8\.0\.0/) and compareVersions($RPS{'Target'}, "8.0.0.7") >= 0) or (isDestinationOneFS(qr/^8\.1\.0/) and compareVersions($RPS{'Target'}, "8.1.0.4") >= 0) or ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.1.1.1") >= 0)) { warning(@impacted, "The above listed interfaces are currently impacted by KB511208. This issue is addressed in OneFS 8.0.0.7, OneFS 8.1.0.4, OneFS 8.1.1.1 and later codes."); } else { fail(@impacted, "The above listed interfaces are currently impacted by KB511208. This issue is addressed in OneFS 8.0.0.7, OneFS 8.1.0.4, OneFS 8.1.1.1 and later codes."); } } # End checkKB511208 $CHECKS{'checkKB519119'}{'Description'} = "Checks if nodes may be impacted by KB 519119"; sub checkKB519119 { return if wasCalled(); return unless $RPS{'Live'}; gatherProductInfo(); return unless $CLUSTER_INFO{'Gen6'}; printCheck("KB519119"); # Gather information from Gen6 nodes my %gather = gatherCluster (qq[2>&1 ${SUDO} isi_for_array -n ].compressRange(@{$CLUSTER_INFO{'Gen6'}}). qq[ ' ]. qq[ echo PS0-A; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b005020000ff10f50050; ]. qq[ echo PS0-B; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b005020000ff10f50060; ]. qq[ echo PS0-C; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b005020000ff10f50070; ]. qq[ echo PS1-A; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b006020000ff10f50050; ]. qq[ echo PS1-B; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b006020000ff10f50060; ]. qq[ echo PS1-C; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b006020000ff10f50070 ]. qq[ ' ]); # Expected RAW IPMI responses my %expected = ("PS0-A" => "b0 00 10 4d 46 52 5f 52 45 53 50 2a 00 00 40 00 00 00 18", "PS0-B" => "b0 00 10 4d 46 52 5f 43 41 4c 90 2a 00 00 00 01 00 00 31", "PS0-C" => "b0 00 10 4d 46 52 5f 46 57 55 90 2b 00 00 20 d4 00 00 1b", "PS1-A" => "b0 00 10 4d 46 52 5f 52 45 53 50 2a 00 00 40 00 00 00 18", "PS1-B" => "b0 00 10 4d 46 52 5f 43 41 4c 90 2a 00 00 00 01 00 00 31", "PS1-C" => "b0 00 10 4d 46 52 5f 46 57 55 90 2b 00 00 20 d4 00 00 1b"); # Analyze each node for issues my %nodes = (); foreach my $lnn (@{$CLUSTER_INFO{'Gen6'}}) { if (!$gather{$lnn}) { missing($lnn); next; } my $cmd = ""; my $response = ""; my $valid_responses = 0; foreach my $line (@{$gather{$lnn}}) { # Gather which PS is being checked $cmd = $1 if $line =~ /(PS[01]-[ABC])/; next unless $cmd and $expected{$cmd}; # Gather raw ipmi response and compare to expected response if ($line =~ /^\s*((?:[0-9a-f]{2}\s)*[0-9a-f]{2})$/) { $response .= ($response?" $1":$1); } else { $response = ""; } next unless $response and $expected{$cmd} eq $response; # Increment the number of valid responses found for this node $cmd = ""; $response = ""; $valid_responses++; } # If there was not 6 responses, the EEPROM may be corrupted debug("checkKB519119: Number of valid responses found on node $lnn: $valid_responses"); if ($valid_responses ne 6) { $nodes{$lnn} = 1; } } # Flag if issues detected if (%nodes) { fail("The following nodes may be impacted by KB 519119: ".compressRange(keys %nodes)); fail("Run the script as outlined in KB 519119 to verify."); fail("Nodes idenfitied as affected by KB 519119 should have the impacted power supply replaced prior to performing any node firmware updates."); } # Output Check result printResults(); } # End checkKB519119 $CHECKS{'checkKB519890'}{'Description'} = "Checks for a known issue with LAGG interfaces in LACP mode when running OneFS 8.0.0.6, 8.0.1.2, 8.1.0.2, and 8.1.1.1"; sub checkKB519890 { return if wasCalled(); # Skip unless upgrading to or running an impacted version return unless ($RPS{'Target'} and $RPS{'Target'} =~ /^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/) or (!$RPS{'Target'} and $CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/); printCheck("KB519890"); gatherNetworkInterfaces(); # Analyze information for issues. my $has_lacp = 0; my @nodes_with_incomplete_lagg = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Network'}; my $has_incomplete_lagg = 0; my @lagg_interfaces = grep {$_ =~ /^lagg/} keys %{$NODE_INFO{$lnn}{'Network'}}; foreach my $lagg_interface (@lagg_interfaces) { next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'} =~ /lacp/; $has_lacp = 1; my $total_ports = 0; my $inactive_ports = 0; foreach my $laggport (@{$NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}}) { my $interface = ""; $interface = $1 if $laggport =~ /^(\S+)/; next unless $interface and $NODE_INFO{$lnn}{'Network'}{$interface}{'status'}; $total_ports++; next if $NODE_INFO{$lnn}{'Network'}{$interface}{'status'} eq "active"; $inactive_ports++; } $has_incomplete_lagg = 1 if $inactive_ports and $total_ports ne $inactive_ports; } push(@nodes_with_incomplete_lagg, $lnn) if $has_incomplete_lagg; } # Fail during pre-checks for upgrades targetting an affected version if ($has_lacp) { $RPS{'KB519890'} = 1 if $CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/ and !hasPatch('patch-(225079|226984)'); if ($RPS{'Target'} and $RPS{'Target'} =~ /^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/) { addCustomerInformation(fail("In OneFS 8.0.0.6, 8.0.1.2, 8.1.0.2, and 8.1.1.1, when some of the ports of a lagg interface in LACP mode go down with other ports still active, OneFS treats the lagg interface as down with No Carrier status. If this lagg is the only interface for front end network, connection to the node from outside the cluster will be lost.")) if $has_lacp; } # If we didn't fail above, warn if the current OneFS version is impacted elsif ($CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/) { addCustomerInformation(warning("In OneFS 8.0.0.6, 8.0.1.2, 8.1.0.2, and 8.1.1.1, when some of the ports of a lagg interface in LACP mode go down with other ports still active, OneFS treats the lagg interface as down with No Carrier status. If this lagg is the only interface for front end network, connection to the node from outside the cluster will be lost.")) if $has_lacp; } } if (@nodes_with_incomplete_lagg) { if ($RPS{'Target'}) { fail("Not all ports of a lagg interface are active on the following nodes which would cause the lagg interface to be down following the OneFS upgrade to $RPS{'Target'}: ".compressRange(@nodes_with_incomplete_lagg)); } else { fail("Not all ports of a lagg interface are active on the following nodes which may cause the lagg interface to be down: ".compressRange(@nodes_with_incomplete_lagg)); } } printResults(); } # End checkKB519890 $CHECKS{'checkKB521890'}{'Description'} = "Checks for criteria outlined in KB 521890"; sub checkKB521890 { return if wasCalled(); # Skip unless upgrading to or running an impacted version (8.0.0.[56], 8.1.0.[123], 8.1.1.[01]) return unless ($RPS{'Target'} and $RPS{'Target'} =~ /^(8\.0\.0\.[56]|8.1.0.[123]|8\.1\.1\.[01])$/) or ($CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.[56]|8\.1\.0\.[123]|8\.1\.1\.[01])$/); printCheck("KB521890"); gatherNetworkInterfaces(); # Analyze information for issues. my $has_lagg = 0; my %nodes_with_ix_lagg = (); my %nodes_with_em_lagg = (); my %nodes_with_igb_lagg = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Network'}; # Issue exists in lagg interfaces my @lagg_interfaces = grep {$_ =~ /^lagg/} keys %{$NODE_INFO{$lnn}{'Network'}}; next unless @lagg_interfaces; # Flag for if the cluster has any lagg ports $has_lagg = 1; # For each lagg interface, check each lagg port for impacted interface types foreach my $lagg_interface (@lagg_interfaces) { next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}; my @laggports = @{$NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}}; my @ix_interfaces = grep {$_ =~ /^ix\d/} @laggports; $nodes_with_ix_lagg{$lnn} = 1 if @ix_interfaces; my @em_interfaces = grep {$_ =~ /^em\d/} @laggports; $nodes_with_em_lagg{$lnn} = 1 if @em_interfaces; my @igb_interfaces = grep {$_ =~ /^igb\d/} @laggports; $nodes_with_igb_lagg{$lnn} = 1 if @igb_interfaces; } } if ($has_lagg) { # There are laggs # Flag issues in destination OneFS versions if ($RPS{'Target'}) { # Upgrading OneFS if ($RPS{'Target'} =~ /^(8\.0\.0\.[56]|8.1.0.[123]|8\.1\.1\.[01])$/) { # Upgrading to an impacted version if (%nodes_with_ix_lagg) { # Has Gen 6 impacted interfaces fail("In the destination OneFS $RPS{'Target'}, the following nodes have a high risk of being impacted by KB 521890: ".compressRange(keys %nodes_with_ix_lagg)); } if (%nodes_with_em_lagg or %nodes_with_igb_lagg) { # Has other Gen impacted interfaces warning("In the destination OneFS $RPS{'Target'}, the following nodes have a very low risk of being impacted by KB 521890: ".compressRange(merge(keys %nodes_with_em_lagg, keys %nodes_with_igb_lagg))); } } if ($RPS{'Target'} =~ /^(8.1.0.[123]|8\.1\.1\.[01])$/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") < 0) { # Upgrading to an impacted OneFS 8.1 code level, perhaps to add Gen6 nodes? addCustomerInformation(fail("After upgrading to OneFS $RPS{'Target'}, avoid configuring 10 GbE Intel network interfaces (ix interfaces) in link aggregation. These interfaces are most commonly seen in Generation 6 nodes. Please review the KB 521890 for details on an issue which impacts 10 GbE Intel network interfaces when used in link aggregation.")); } } # Flag issues in current OneFS versions if ($CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.[56]|8\.1\.0\.[123]|8\.1\.1\.[01])$/) { # Running an impacted version if ($RPS{'Target'} and $RPS{'Target'} !~ /^(8\.0\.0\.[56]|8.1.0.[123]|8\.1\.1\.[01])$/ and %nodes_with_ix_lagg) { critical("In the current OneFS $CLUSTER_INFO{'OneFS'}, the following nodes have a high risk of being impacted by KB 521890: ".compressRange(keys %nodes_with_ix_lagg)); # Toggle a flag that will be used to recommend against node firmware or reboot activities $RPS{'KB521890'} = 1; } else { if (%nodes_with_ix_lagg) { # Has Gen 6 impacted interfaces fail("In the current OneFS $CLUSTER_INFO{'OneFS'}, the following nodes have a high risk of being impacted by KB 521890: ".compressRange(keys %nodes_with_ix_lagg)); # Toggle a flag that will be used to recommend against node firmware or reboot activities $RPS{'KB521890'} = 1; } if (%nodes_with_em_lagg or %nodes_with_igb_lagg) { # Has other Gen impacted interfaces warning("In the current OneFS $CLUSTER_INFO{'OneFS'}, the following nodes have a very low risk of being impacted by KB 521890: ".compressRange(merge(keys %nodes_with_em_lagg, keys %nodes_with_igb_lagg))); } if (!%nodes_with_ix_lagg and $CLUSTER_INFO{'OneFS'} =~ /^(8\.1\.0\.[123]|8\.1\.1\.[01])$/) { # Has no impacted interfaces, but might be adding Generation 6 nodes addCustomerInformation(warning("If planning to add Generation 6 nodes, please first review the KB 521890 for details on an issue which impacts 10 GbE (ix) network interfaces on Generation 6 nodes when used in link aggregation.")); } } } } printResults(); } # End checkKB521890 ### # checkLACPSFP: Checks for LACP on cxgb interfaces for KB 489262 $CHECKS{'checkLACPSFP'}{'Description'} = "Checks for LACP on cxgb interfaces for KB 489262"; sub checkLACPSFP { return if wasCalled(); # return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0; return unless $RPS{'Target'} and $RPS{'Target'} =~ /^8\.0\.0\.[012345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/; gatherNetworkInterfaces(); printCheck("LACP over CXGB interfaces with TwinAx SFP"); debug("checkLACPSFP: Checking for network pools configured with LACP aggregation mode"); my @nodes_affected = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$lnn}{'Network'}; foreach my $interface (keys %{$NODE_INFO{$lnn}{'Network'}}) { next unless $interface; if ($interface =~ /cxgb/ and $NODE_INFO{$lnn}{'Network'}{$interface}{'media'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggdev'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'media'} =~ /twinax/i and $NODE_INFO{$lnn}{'Network'}{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggdev'}}{'laggproto'} and $NODE_INFO{$lnn}{'Network'}{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggdev'}}{'laggproto'} =~ /lacp/i) { debug("checkLACPSFP: !!! Network interface $interface on node $lnn may be impacted"); push(@nodes_affected, $lnn) unless grep {$_ eq $lnn} @nodes_affected; } if ($NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} =~ /lacp/i) { foreach my $cxgb_interface (grep {$_ =~ /^cxgb/} @{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'}}) { next unless $cxgb_interface; if($NODE_INFO{$lnn}{'Network'}{$cxgb_interface}{'media'} and $NODE_INFO{$lnn}{'Network'}{$cxgb_interface}{'media'} =~ /twinax/i) { debug("checkLACPSFP: !!! Network interface $interface on node $lnn may be impacted"); push(@nodes_affected, $lnn) unless grep {$_ eq $lnn} @nodes_affected; } } } } } if (@nodes_affected) { # FAIL message fail("The following nodes were found as having Chelsio NICs using TwinAx SFP transceivers that are part of an aggregate network configured as LACP: ".compressRange(@nodes_affected)); fail("An issue exists in OneFS 8.x releases where communication over the associated aggregate network interfaces may fail to work after an upgrade to impacted OneFS 8.x versions"); fail("A fix for this issue is targeted for OneFS 8.0.0.6, OneFS 8.0.1.2, and OneFS 8.1.0.1."); fail("A directed availability patch is available upon request and approval of Isilon support SMEs, however, if the only connectivity to the cluster is via impacted interfaces, this may leave the cluster in an unreachable state until the patch has been applied."); fail("Isilon Certified SFP transceivers are also known to work, review the 'SFP transceivers for Generation 4 and Generation 5 nodes' section of the 'Isilon Supportability and Compatibility Guide'."); # Information appended to customer information addCustomerInformation ("TwinAx SFP, LAGG, & LACP", "The following nodes were found as having Chelsio NICs using TwinAx SFP transceivers that are part of an aggregate network configured as LACP: ".compressRange(@nodes_affected), "An issue exists in OneFS 8.x releases where communication over the associated aggregate network interfaces may fail to work after an upgrade to impacted OneFS 8.x versions", "A fix for this issue is targeted for OneFS 8.0.0.6, OneFS 8.0.1.2, and OneFS 8.1.0.1.", "A directed availability patch is available upon request and approval of Isilon support SMEs, however, if the only connectivity to the cluster is via impacted interfaces, this may leave the cluster in an unreachable state until the patch has been applied.", "Isilon Certified SFP transceivers are also known to work, review the 'SFP transceivers for Generation 4 and Generation 5 nodes' section of the 'Isilon Supportability and Compatibility Guide'."); } printResults(); } # End checkLACPSFP $CHECKS{'checkMemory'}{'Description'} = "Checks each DIMM to meet criteria outlined in KB89614 and if the expected (per product info line) matches closely the reported RAM."; sub checkMemory { return if wasCalled(); checkHardwareStatus(); gatherProductInfo(); printCheck("Memory"); checkExpectedMemory(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array '${SUDO} isi_dmilog|grep DIMM'; sleep 2; 2>&1 ${SUDO} isi_for_array '${SUDO} isi_hwmon -l|grep DIMM'"); } else { %gather = gatherCluster("2>&1 grep -H DIMM */isi_dmilog"); } # Analyze information for issues. my %dimm_report = (); foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - checkMemory"); next unless $gather{$lnn}; # Check for DIMM errors my %dimm_errors = (); foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /^\s+(\d+)\/(\d+)\/(\d+)\s+(\d+):(\d+):(\d+)\s+\S+\s+(\S.*?):\s+(\d+)\s+times? on\s+(\S+)$/; debug("checkMemory: Node $lnn: $line"); my $month = $1; my $day = $2; my $year = $3; my $hour = $4; my $min = $5; my $sec = $6; my $type = $7; my $count = $8; my $dimm = $9; # Sometimes isi_dmilog has reports on 01/00/00 0:00:00.. ignoring these because there is no time stamp next if ($day == 0); # Fix a bug where dmi log rolls over the date by max int size * 2 # Also assign epoch time for the error to $time my $time = 0; if ($year > 100) { eval { $time = timelocal($sec, $min, $hour, $day, $month - 1, $year + 1832); }; warning($@) if $@; next if !$time; $time += 1006623105; ($sec, $min, $hour, $day, $month, $year) = localtime($time); $year += 1900; $month += 1; debug("checkMemory: Time conversion performed, actual date & time: $month/$day/$year $hour:$min:$sec"); } else { $year += 2000; eval { $time = timelocal($sec, $min, $hour, $day, $month - 1, $year); }; warning($@) if $@; next if !$time; } verbose("DIMM: $dimm, Count: $count, Type: $type, Time: $month/$day/$year $hour:$min:$sec"); # Skip if more than 1 year old next if ($RPS{'Runtime'} - $time >= 30758400); next if ($RPS{'Runtime'} - $time >= 7776000); $dimm_report{$lnn}{$dimm}{$time}{'Type'} = $type; $dimm_report{$lnn}{$dimm}{$time}{'Count'} = $count; my $date_str = sprintf("%04d-%02d-%02d", $year, $month, $day); $dimm_errors{$dimm}{$type}{$date_str} += $count; } foreach my $dimm (sort keys %dimm_errors) { next unless $dimm_errors{$dimm}; foreach my $type (sort keys %{$dimm_errors{$dimm}}) { next unless $dimm_errors{$dimm}{$type}; my $str = "Node $lnn DIMM $dimm is reporting $type with the following dates(count):"; foreach my $date (sort keys %{$dimm_errors{$dimm}{$type}}) { $str .= " ${date}($dimm_errors{$dimm}{$type}{$date})" } warning($str); } } } foreach my $lnn (sort {$a <=> $b} keys %dimm_report) { # Different generations of hardware have different allowances for DIMM errors in a 24 hour period my $allowed = 500; if ($NODE_INFO{$lnn}{'Generation'}) { if ($NODE_INFO{$lnn}{'Generation'} eq "Gen4") { $allowed = 15; } elsif ($NODE_INFO{$lnn}{'Generation'} eq "Gen5") { $allowed = 500; } elsif ($NODE_INFO{$lnn}{'Generation'} eq "Gen6") { $allowed = 5000; } } my @dimms_with_issues = (); foreach my $dimm (keys %{$dimm_report{$lnn}}) { my @under_24_hours = (); my $fatal = 0; my $over_allowed_in_24_hours = 0; foreach my $time (sort keys %{$dimm_report{$lnn}{$dimm}}) { # Add new time push(@under_24_hours, $time); # Shift off old times shift(@under_24_hours) while ($time - $under_24_hours[0] > 86400); # Fatal errors!? $fatal = 1 if not $dimm_report{$lnn}{$dimm}{$time}{'Type'} =~ /correctable|non-fatal/i; # Maximum of 15 errors on a DIMM allowed within a 24-hour period my $error_count = 0; $error_count += $dimm_report{$lnn}{$dimm}{$_}{'Count'} foreach (@under_24_hours); $over_allowed_in_24_hours = 1 if $error_count > $allowed; } push(@dimms_with_issues, $dimm) if $fatal or $over_allowed_in_24_hours; } fail("Node $lnn has DIMMs that met criteria for replacement: ".join(",",@dimms_with_issues)) if @dimms_with_issues; } info("Refer to KB210516 (https://support.emc.com/kb/210516) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkMemory ### # checkMirrors: Boot Drives - Cannot perform upgrade with degraded boot drive # https://support.emc.com/kb/88746 - Cannot perform upgrade with degraded boot drive $CHECKS{'checkMirrors'}{'Description'} = "Checks the boot mirror health"; sub checkMirrors { return if wasCalled(); gatherProductInfo(); # Skip if virtual cluster return if $CLUSTER_INFO{'Virtual'}; printCheck("Mirror Status"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array gmirror status"); } else { %gather = gatherCluster("2>&1 grep -H . */gmirror_status"); } # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); next if $NODE_INFO{$lnn}{'Accelerator'}; next if $NODE_INFO{$lnn}{'Generation'} and $NODE_INFO{$lnn}{'Generation'} eq "IQ"; if (!$gather{$lnn}) { missing($lnn); next; } my $name =""; my $status = ""; my %mirrors = (); foreach my $line (@{$gather{$lnn}}) { verbose($line); next if ($line =~ /\s*Name\s*Status\s*Components/); if ($line =~ /(mirror\/\S+)\s+(\S+)\s+(\S+)\s*/) { $name = $1; $mirrors{$name}{'status'} = $2; push(@{$mirrors{$name}{'components'}}, $3); } elsif ($line =~ /^\s+(\S+)/) { push(@{$mirrors{$name}{'components'}}, $1) if $mirrors{$name}; } } if (%mirrors) { $NODE_INFO{$lnn}{'Bootflash'} = 1; debug("NODE_INFO->$lnn->Bootflash=1"); } my @mirrors_missing_components = (); my @mirrors_degraded = (); foreach my $name (keys %mirrors) { my $components = scalar(@{$mirrors{$name}{'components'}}); debug("checkMirrors: Node $lnn: Mirror: $name has a status: $mirrors{$name}{'status'} and $components components"); # Degraded mirrors if ($mirrors{$name}{'status'} eq "DEGRADED") { push(@mirrors_degraded, $name); } # Only one component if($components < 2 and $name !~ /crash|mfg|keystore|dump/ and not $CLUSTER_INFO{'Virtual'}) { push(@mirrors_missing_components, $name); } } fail("Node $lnn is reporting degraded for the following mirrors: ".join(",", @mirrors_degraded)) if (@mirrors_degraded); fail("Node $lnn is reporting only one component for the following mirrors: ".join(",", @mirrors_missing_components)) if (@mirrors_missing_components); } info("Refer to KB210515 (https://support.emc.com/kb/210515) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkMirrors $CHECKS{'checkNodeCompatibility'}{'Description'} = "Checks node compatibility by comparing it against known supported versions."; sub checkNodeCompatibility { return if wasCalled(); return unless $RPS{'Target'}; gatherProductInfo(); checkPartitions() if $CLUSTER_INFO{'IQ'}; printCheck("Node Compatibility"); my %CompatibilityMatrix = ( # IQ Node types "IQ 1920e" => "5.5", "IQ 3000e" => "5.5", "IQ 4800e" => "5.5", "IQ 6000e" => "5.5", "IQ 200" => "5.5", "IQ 1920i" => "6.5", "IQ 3000i" => "6.5", "IQ 6000i" => "6.5", "IQ 9000i" => "6.5", "IQ 12000i" => "6.5", "IQ 1920x" => "7.1", "IQ 3000x" => "7.1", "IQ 6000x" => "7.1", "IQ 9000x" => "7.1", "IQ 36NL" => "7.1", "IQ 72NL" => "7.1", "IQ 12000x" => "7.1", "IQ 10000x-ssd" => "8.0.0.6", "IQ 32000x-ssd" => "8.0.0.6", "IQ 36000x" => "8.0.0.6", "IQ 72000x" => "8.0.0.6", "IQ Accelerator-x" => "8.0.0.6", "Accelerator" => "8.0.0.6", "IQ 5000s-ssd" => "8.0.0.6", "IQ 5400s" => "8.0.0.6", "IQ 108NL" => "8.0", # Gen 4 "X200" => "8.1", "S200" => "8.1", "X400" => "8.1", "NL400" => "8.1", "A100" => "TBD", # Gen 5 "HD400" => "TBD", "S210" => "TBD", "X210" => "TBD", "X410" => "TBD", "NL410" => "TBD", # Gen 6 "F800" => "TBD", "F810" => "TBD", "H600" => "TBD", "H6000" => "TBD", "H500" => "TBD", "H400" => "TBD", "A200" => "TBD", "A2000" => "TBD", # Virtual "IQ VMWare" => "8.1", "V200" => "TBD", # Isilon SD Edge virtual nodes "V100" => "8.1", ); # Iterate through each node verboseHeader("Node Compatibility"); verbose(sprintf("%15s %10s %s", "Node Type", "Max OneFS", "Product")); verbose(sprintf("%15s %10s %s", "-"x15, "-"x10, "-"x10)); foreach my $lnn (@LNNS) { # Node Product details must be available next unless $NODE_INFO{$lnn}{'Product'}; my $product = $NODE_INFO{$lnn}{'Product'}; # Search for the node type in the compatibility matrix my ($type) = grep {$product =~ /$_/i} sort keys %CompatibilityMatrix; if (!$type) { # The node type was not found in the compatibility matrix verbose(sprintf("%15s %10s %s", "Unknown", "Unknown", $product)); fail("Unable to find a match for $product"); fail("Script Error, please contact EMC RPS Isilon DET team"); next; } # The node type was found in the compatibility matrix # Store the maximum OneFS version my $max = $CompatibilityMatrix{$type}; verbose(sprintf("%15s %10s %s", $type, $max, $product)); # Skip to the next node if there is no max next if $max eq "TBD"; # Perform compatibility checks if ($NODE_INFO{$lnn}{'Series'} and $NODE_INFO{$lnn}{'Series'} =~ /i_series/ and compareVersions($RPS{'Target'}, "7.0") >= 0 and compareVersions($max, "7.0") >= 0) { # i Series nodes are not supported in OneFS 7.0 or later fail("Node $lnn is an i series node and is not compatible with OneFS 7.0 or later"); } elsif (compareVersions($RPS{'Target'}, $max) > 0) { # The destination OneFS is greater than the maximum supported version for that node type fail("Node $lnn of type $product is compatible up to OneFS $max"); } elsif ($RPS{'Root less than 500MB'}{$lnn} and (($RPS{'Target'} =~ /^8\.0\.0/ and compareVersions($RPS{'Target'}, "8.0.0.7") >= 0) or (compareVersions($RPS{'Target'}, "8.1.0") >= 0))) { # The node has less than 500MB total in / fail("Node $lnn does not have a sufficient root partition to support OneFS $RPS{'Target'}"); } } info("Refer to KB210517 (https://support.emc.com/kb/210517) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkNodeCompatibility $CHECKS{'checkNodes'}{'Description'} = "Checks if nodes are EOSL"; sub checkNodes { return if wasCalled(); printCheck("Node EOSL"); gatherProductInfo(); # Analyze information for issues. my @eosl = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Product'}; # All IQ Nodes are EOSL if ($NODE_INFO{$lnn}{'Product'} =~ /^IQ/) { push(@eosl, $lnn); } } # Flag if any nodes were identified as EOSL warning("The following nodes are EOSL: ".compressRange(@eosl)) if @eosl; printResults(); } # End checkNodes ### # gatherBMCInfo: Gather BMC information sub gatherBMCInfo { return if wasCalled(); my $nodes = compressRange(nodesOfType('^(S210|X210|X410|HD400|NL410)')); return unless $nodes; my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array -n $nodes time ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -d -V -a bmc|grep firmware"); } else { %gather = gatherCluster("2>&1 grep -H firmware */bmc_info"); } foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless ($gather{$lnn}); my ($firmware) = grep(/IPMI firmware version = (.*)$/, @{$gather{$lnn}}); if ($firmware and $firmware =~ /IPMI firmware version = (.*)$/) { $NODE_INFO{$lnn}{'Firmware'}{'BMC'} = $1; debug("NODE_INFO->${lnn}->Firmware->BMC=$1"); } } } # End gatherBMCInfo ### # gatherBXEDriver: Gather BXE NIC driver version. Only works on live clusters. sub gatherBXEDriver { return if wasCalled(); my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array sysctl dev.bxe.0.version"); } else { %gather = gatherCluster('2>&1 zgrep -H "bxe0.*BXE.*ISILON" */varlog.tar/log/messages*'); } return unless %gather; foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { foreach my $line (@{$gather{$lnn}}) { if ($line =~ /^dev.bxe.0.version:\s*(\S+)$/) { $NODE_INFO{$lnn}{'dev.bxe.0.version'} = $1; debug("NODE_INFO->${lnn}->dev.bxe.0.version=$1"); } elsif ($line =~ /BXE v:(.*?)>/) { next if ($NODE_INFO{$lnn}{'dev.bxe.0.version'} and compareVersions($NODE_INFO{$lnn}{'dev.bxe.0.version'}, $1) >= 0); $NODE_INFO{$lnn}{'dev.bxe.0.version'} = $1; debug("NODE_INFO->${lnn}->dev.bxe.0.version=$1"); } } } } # End gatherBXEDriver ### # gatherCMCInfo: Gather CMC information sub gatherCMCInfo { return if wasCalled(); my $nodes = compressRange(nodesOfType('^(S210|X210|X410|HD400|NL410)')); return unless $nodes; my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array -n $nodes time ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -d -V -a cmc|grep firmware"); } else { %gather = gatherCluster("2>&1 grep -H firmware */cmc_info"); } foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless ($gather{$lnn}); my ($firmware) = grep(/IPMI firmware version = (.*)$/, @{$gather{$lnn}}); if ($firmware and $firmware =~ /IPMI firmware version = (.*)$/) { $NODE_INFO{$lnn}{'Firmware'}{'CMC'} = $1; debug("NODE_INFO->${lnn}->Firmware->CMC=$1"); } } } # End gatherCMCInfo ### # gatherCTO: Gathers CTO Capable node information sub gatherCTO { return if wasCalled(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi_inventory_tool --help | grep CTO"); } else { %gather = gatherCluster("2>&1 grep -H . */asbuilt.xml"); } # Analyze information for issues foreach my $node (sort {$a <=> $b} keys %NODE_INFO) { if (!$gather{$node}) { missing($node); next; } my $cto = 1; foreach my $line (@{$gather{$node}}) { $cto = 0 if ($line =~ /Tool is only supported on CTO capable nodes/); $cto = 0 if ($line =~ /this node is not CTO capable/); } $NODE_INFO{$node}{'CTO'} = $cto; debug("NODE_INFO->$node->CTO->$cto"); } } # End gatherCTO ### # gatherProductInfo: Gathers from isi_hw_status -i the Serial Number and Product Identifier sub gatherProductInfo { return if wasCalled(); # Gather information my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi_hw_status -i"); } else { %gather = gatherCluster("2>&1 grep -H . */isi_hw_status"); } # Analyze information foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { if ($line =~ /^\s*Product: (.*)$/) { $NODE_INFO{$lnn}{'Product'} = $1; debug("NODE_INFO->${lnn}->Product=$1"); if ($NODE_INFO{$lnn}{'Product'} =~ /^V|VMWare/) { debug("NODE_INFO->${lnn}->Generation=Virtual"); $NODE_INFO{$lnn}{'Generation'} = "Virtual"; push(@{$CLUSTER_INFO{'Virtual'}},$lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ /^IQ/) { debug("NODE_INFO->${lnn}->Generation=IQ"); $NODE_INFO{$lnn}{'Generation'} = "IQ"; push(@{$CLUSTER_INFO{'IQ'}},$lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ /^(S200|X400|X200|NL400)/) { debug("NODE_INFO->${lnn}->Generation=Gen4"); $NODE_INFO{$lnn}{'Generation'} = "Gen4"; push(@{$CLUSTER_INFO{'Gen4'}},$lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ /^A100/) { push(@{$CLUSTER_INFO{'A100'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ /^(S210|X410|X210|NL410|HD400)/) { debug("NODE_INFO->${lnn}->Generation=Gen5"); $NODE_INFO{$lnn}{'Generation'} = "Gen5"; push(@{$CLUSTER_INFO{'Gen5'}},$lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ /^(A200|A2000|H400|H500|H600|F800)/) { debug("NODE_INFO->${lnn}->Generation=Gen6"); $NODE_INFO{$lnn}{'Generation'} = "Gen6"; push(@{$CLUSTER_INFO{'Gen6'}},$lnn); } else { debug("NODE_INFO->${lnn}->Generation=Unknown"); $NODE_INFO{$lnn}{'Generation'} = "Unknown"; } if ($NODE_INFO{$lnn}{'Product'} =~ /^A100|Accelerator/) { debug("NODE_INFO->${lnn}->Accelerator=1"); $NODE_INFO{$lnn}{'Accelerator'} = 1; push(@{$CLUSTER_INFO{'Accelerator'}}, $lnn); } } elsif ($line =~ /^\s*Series: (.*)$/) { $NODE_INFO{$lnn}{'Series'} = $1; debug("NODE_INFO->${lnn}->Series=$1"); } elsif ($line =~ /^\s*Config: (.*)$/) { $NODE_INFO{$lnn}{'Config'} = $1; debug("NODE_INFO->${lnn}->Config=$1"); } elsif ($line =~ /^\s*SerNo: (.*)$/) { $NODE_INFO{$lnn}{'Serial'} = $1; debug("NODE_INFO->${lnn}->Serial=$1"); } elsif ($line =~ /^\s*ChsSlot: (.*)$/) { $NODE_INFO{$lnn}{'Chassis Slot'} = $1; debug("NODE_INFO->${lnn}->Chassis Slot=$1"); } } } } # End gatherProductInfo ### # addCustomerInformation: Adds information to be provided to the customer # while attempting to ensure information is unique. sub addCustomerInformation { return unless @_; # Default to Generic section my $section = "Generic"; # If additional arguments, set section to first argument if (scalar(@_) > 1) { $section = shift; } # Merge information into section if ($RPS{'Customer Information'}{$section}) { push(@{$RPS{'Customer Information'}{$section}}, $_) foreach (@_); } else { @{$RPS{'Customer Information'}{$section}} = @_; } } # End addCustomerInformation ### # addRequiredFiles: Adds files to the list of required files, making # sure to merge with existing files to avoid any duplicates. sub addRequiredFiles { # Initialize the array of required files @{$RPS{'Required Files'}} = () unless $RPS{'Required Files'}; # Add to the list of required files @{$RPS{'Required Files'}} = merge(@{$RPS{'Required Files'}}, @_); } # End addRequiredFiles ### # compareVersions: Compares two OneFS versions. # $1 > $2 => Positive # $1 = $2 => 0 # $1 < $2 => Negative # If only a partial version is provided, it will match up to given criteria. # IE: 7.0 and 7.0.2 would be equivalent. sub compareVersions { my @left = split(/\.|-/, $_[0]); my @right = split(/\.|-/, $_[1]); my $last = ($#left > $#right)?$#right:$#left; my $comparison = 0; for (my $i = 0; $i <= $last; $i++) { if ($left[$i] =~ /^\d*$/ and $right[$i] =~ /^\d*$/) { $comparison = $left[$i] - $right[$i]; } else { $comparison = $left[$i] cmp $right[$i]; } last if ($comparison); } debug("compareVersions: Arguments: $_[0], $_[1] Results: $comparison"); return $comparison; } # End compareVersions ### # compressRange: Takes an array of nodes and returns a compressed range. sub compressRange { return "" unless @_; debug("compressRange: Argument: [ ".join(",",@_)." ]"); my $compressed_range = ""; my @expanded_range = sort {$a <=> $b} @_; my $last = shift @expanded_range; my $start = $last; foreach my $current (@expanded_range) { if ($last + 1 != $current) { $compressed_range .= "," if ($compressed_range); if ($start == $last) { $compressed_range .= "$start"; } elsif ($start + 1 == $last) { $compressed_range .= "$start,$last"; } else { $compressed_range .= "$start-$last"; } $start = $current; } $last = $current; } $compressed_range .= "," if ($compressed_range); if ($start == $last) { $compressed_range .= "$start"; } elsif ($start + 1 == $last) { $compressed_range .= "$start,$last"; } else { $compressed_range .= "$start-$last"; } debug("compressRange: Return: $compressed_range"); return $compressed_range; } # End compressRange ### # convertDurationToSeconds: Convert from a duration, i.e. 1Y1W1D1H1m1s to 123153424 seconds sub convertDurationToSeconds { return 0 unless @_ and $_[0]; my $duration = $_[0]; my $seconds = 0; if ($duration =~ /(\d+)Y/) { $seconds += $1 * 31536000; } if ($duration =~ /(\d+)M/) { $seconds += $1 * 2592000; } if ($duration =~ /(\d+)W/) { $seconds += $1 * 604800; } if ($duration =~ /(\d+)D/) { $seconds += $1 * 86400; } if ($duration =~ /(\d+)H/) { $seconds += $1 * 3600; } if ($duration =~ /(\d+)m/) { $seconds += $1 * 60; } if ($duration =~ /(\d+)s/) { $seconds += $1; } debug("convertDurationToSeconds: Arguments: $duration Results: $seconds"); return $seconds; } # End convertDurationToSeconds ### # convertFromNumeric: Convert to a number/quantifier from a numeric, provided a quantifier to convert to. sub convertFromNumeric { return unless $_[0] and $_[1]; return 0 unless $_[0] =~ /^\d+$/; return 0 unless $_[1] =~ /^[KMGTP]$/; my $number = 0; if ($_[1] eq "K") { $number = $_[0] / 2**10; } elsif ($_[1] eq "M") { $number = $_[0] / 2**20; } elsif ($_[1] eq "G") { $number = $_[0] / 2**30; } elsif ($_[1] eq "T") { $number = $_[0] / 2**40; } elsif ($_[1] eq "P") { $number = $_[0] / 2**50; } $number = int($number) + 1; debug("convertFromNumber: Arguments: $_[0] $_[1] Results: ${number}${_[1]}B"); return "${number}${_[1]}B"; } # End convertFromNumeric ### # converIPToNumeric: Converts a string IP into it's numerical equivalent sub convertIPToNumeric { return unless $_[0]; my $str_ip = $_[0]; my @str_octets = split(/\./, $str_ip); my $int_ip = 0b; if ($#str_octets != 3) { # Not really an IPv4 IP return; } for(my $i = 3; $i >= 0; $i--) { $int_ip += $str_octets[$i] * (2 ** ((3-$i) * 8)); } debug("convertIPToNumeric: Arguments: $str_ip Return: ".sprintf("%032b", $int_ip)); return $int_ip; } # End convertIPToNumeric ### # convertToNumeric: Convert from a number and quantifier into just a number sub convertToNumeric { return 0 unless $_[0] =~ /^\s*(\d*\.?\d*)\s*([KMGTP])\D*\s*$/; my $numeric = $1; my $quantifier = $2; return 0 unless $numeric and $quantifier; if ($quantifier eq "K") { $numeric *= 2**10; } elsif ($quantifier eq "M") { $numeric *= 2**20; } elsif ($quantifier eq "G") { $numeric *= 2**30; } elsif ($quantifier eq "T") { $numeric *= 2**40; } elsif ($quantifier eq "P") { $numeric *= 2**50; } debug("convertToNumeric: Arguments: $_[0] Results: $numeric"); return $numeric; } # End convertToNumeric ### # critical: Record critical related criticalrmation, but only if enabled for critical sub critical { push(@CRITICAL, @_) if @_; return @_; } # End critical ### # debug: Record debug related information, but only if enabled for debug sub debug { push(@DEBUG, @_) if @_ and $RPS{'Debug'}; return @_; } # End debug ### # details: Record details related information, but only if enabled for details sub details { push(@DETAILS, @_) if @_; return @_; } # End details ### # expandRange: Takes a node list such as 1,3,5-6,11-132,2 and generates an array of all nodes contained sub expandRange { return () unless $_[0]; debug("expandRange: Argument: $_[0]"); my @expanded_range = (); my @split_ranges = split(",", $_[0]); foreach my $range (@split_ranges) { my ($start, $end) = split("-", $range); next unless $start; if (!$end) { push(@expanded_range, $start); } else { push(@expanded_range, ($start .. $end)); } } debug("expandRange: Return: [ ".join(",", @expanded_range)." ]") if @expanded_range; return @expanded_range; } # End expandRange ### # fail: Record fail related failrmation, but only if enabled for fail sub fail { push(@FAIL, @_) if @_; return @_; } # End fail ### # gatherCluster: Runs command and parses based on it being a cluster wide command. sub gatherCluster { my %gather = (); debug("Gathering: $_[0]"); my @response = qx{$_[0]}; foreach (@response) { s/\x1b\[[0-9;]*m//g; next if (/No such file or directory/); next if (/command not found:/); push(@{$gather{$2}}, $3) if (/^(\S+)-(\d+)\/\S*?:(.*?)\s*$/); push(@{$gather{$2}}, $3) if (/^(\S+)-(\d+): (.*?)\s*$/); } foreach my $lnn (keys %gather) { next unless $gather{$lnn}; debug("--------------------", " Node $lnn", "--------------------"); debug(@{$gather{$lnn}}); } return %gather; } # End gatherCluster ### # gatherLocal: Runs command and parses based on it being a local only command. sub gatherLocal { my @gather = (); debug("Gathering: $_[0]"); my @response = qx{$_[0]}; foreach (@response) { s/\x1b\[[0-9;]*m//g; next if (/No such file or directory/); next if (/command not found:/); push(@gather, $1) if (/^(.*?)\s*$/); } return debug(@gather); } # End gatherLocal ### # generateUnpackTemplate: Generates an unpack template based on provided header line and header labels sub generateUnpackTemplate { debug("generateUnpackTemplate: ".join(",",@_)); if (scalar(@_) <= 1) { debug("generateUnpackTemplate: Not enough arguments provided"); return ""; } my $unpack_template = ""; my $header_line = shift; my @indexes = (); foreach my $label (@_) { push(@indexes, index($header_line, $label)); } my $previous_index = 0; foreach my $index (sort {$a <=> $b} @indexes) { next if ($index == -1 or $index == 0); my $unpack_diff = $index - $previous_index - 1; $unpack_template .= "A${unpack_diff}x"; $previous_index = $index; } $unpack_template .= "A*"; debug("generateUnpackTemplate: $unpack_template"); return $unpack_template; } # End generateUnpackTemplate ### # getCoreCount: Takes the PROC line from isi_hw_status and return the number of cores (int). sub getCoreCount { return 0 unless $_[0]; $_ = $_[0]; s/-HT//; s/Single/1/; s/Dual/2/; s/Quad/4/; s/Hexa/6/; s/Octa/8/; my ($proc, $core) = (1, 1); $proc = $1 if /(\d+)-proc/; $core = $1 if /(\d+)-core/; return int($proc * $core); } # End getCoreCount ### # info: Record info related information, but only if enabled for info sub info { push(@INFO, @_) if @_; return @_; } # End info ### # Initiate health check script, header and stuff. sub initRPSisiHealth { $RPS{'Runtime'} = time(); # Parse command line arguments parseArguments(@ARGV); # The version of this script is over 30 days old my $days_since_release = int(($RPS{'Runtime'} - $RPS{'Release Timestamp'}) / 86400); if ($days_since_release > 30) { printUrgent("This version is over $days_since_release days old, a new version may be available."); printUrgent("If FTP access is available, download the latest version with:"); printUrgent(" curl --disable-epsv -O ftp.emc.com/pub/rcm/Isilon/tools/IOCA"); } # Evaluate whether live on a cluster or a logset, if a logset, # determine when the gather was performed $RPS{'Live'} = whereRunning(); gatherPackageInfo(); # Display current time for live cluster or logset path for a logset if ($RPS{'Live'}) { printCheck("Live Cluster Analysis", scalar(localtime($RPS{'Runtime'}))); $RPS{'Timestamp'} = $RPS{'Runtime'}; } else { printCheck("Logset Path", getcwd()); # Output gather date if ($RPS{'Gather Date'}) { printCheck("Gather Date", scalar(localtime($RPS{'Gather Date'}))); $RPS{'Timestamp'} = $RPS{'Gather Date'}; } else { printCheck("Gather Date", "Undetermined"); } # Warn if logset gather date is over 1 month old if ($RPS{'Gather Date'} and $RPS{'Runtime'} - $RPS{'Gather Date'} > 2592000) { printUrgent("Logset is over 1 month old"); } } # Gather basic node and cluster information gatherArrayXML(); if (!$CLUSTER_INFO{'Name'}) { if ($RPS{'Live'}) { printUrgent("Unable to gather details from /etc/ifs/array.xml"); } else { printUrgent("Unable to gather details from */etc.tar/etc/ifs/array.xml */etcifs.tar/ifs/array.xml");; printUrgent("Please ensure you are running health checks with a full gather"); } exit; } # Determine if the cluster is compliance mode and assign sudo variable gatherComplianceMode(); $SUDO = ($CLUSTER_INFO{'Compliance'})?"sudo ":""; # Check that script is being ran by root or compadmin if ($RPS{'Live'}) { my $expected_user = ($CLUSTER_INFO{'Compliance'}?'compadmin':'root'); chomp(my $current_user = `whoami`); if ($current_user ne $expected_user) { printUrgent("IOCA should be ran as $expected_user"); printUrgent("Results may not be accurate"); } } printCheck("Cluster Name", $CLUSTER_INFO{'Name'}) if $CLUSTER_INFO{'Name'}; printCheck("Cluster GUID", $CLUSTER_INFO{'GUID'}) if $CLUSTER_INFO{'GUID'}; printCheck("Node Count", $CLUSTER_INFO{'Node Count'}) if $CLUSTER_INFO{'Node Count'}; printCheck("SmartLock Compliance Mode", "ENABLED") if $CLUSTER_INFO{'Compliance'}; # Gather OneFS version details gatherOneFSVersions(); printCheck("Current OneFS Version", $CLUSTER_INFO{'OneFS'}) if $CLUSTER_INFO{'OneFS'}; # Check destination OneFS version checkDestinationOneFS(); # Check OneFS Version for issues # This script will not work on OneFS 5.5 and earlier printUrgent("IOCA is not compatible with OneFS 5.5 and earlier versions\n") if ($CLUSTER_INFO{'OneFS'} and $CLUSTER_INFO{'OneFS'} =~ /^5\.5/); # Exit the script if OneFS Version 5.5 or earlier exit if ($CLUSTER_INFO{'OneFS'} and compareVersions($CLUSTER_INFO{'OneFS'}, "5.5") <= 0); # If a specific check is to be ran, run it and exit if ($RPS{'Run'}) { &{$_}() foreach @{$RPS{'Run'}}; if (@INFO or @DEBUG or @DETAILS or @WARN or @FAIL) { printCheck("Script clean-up"); printResults(); } print toJSON(\@RESULTS)."\n" if @RESULTS and $RPS{'JSON'}; exit; } } # End initRPSisiHealth ### # isCurrentOneFS: Uses either Regexp or scalar to test current OneFS sub isCurrentOneFS { gatherOneFSVersions(); return 0 unless $_[0]; return 0 unless $CLUSTER_INFO{'OneFS'}; if (ref($_[0]) eq "Regexp") { return $CLUSTER_INFO{'OneFS'} =~ $_[0]; } else { return $CLUSTER_INFO{'OneFS'} eq $_[0]; } } # End isCurrentOneFS ### # isDestinationOneFS: Uses either Regexp or scalar to test destination OneFS sub isDestinationOneFS { return 0 unless $_[0]; return 0 unless $RPS{'Target'}; if (ref($_[0]) eq "Regexp") { return $RPS{'Target'} =~ $_[0]; } else { return $RPS{'Target'} eq $_[0]; } } # End isDestinationOneFS ### # isResults: Return 1 if there are results to print sub isResults { if (@FAIL or @CRITICAL or @WARN or @INFO or @DETAILS or @DEBUG or %MISSING) { return 1; } return 0; }# End isResults ### # Given two arrays, combine them removing duplicates and return the merged array sub merge { my %hash = (); $hash{$_} = 1 foreach(@_); return keys %hash; } ### # missing: Store a list of nodes missing information sub missing { return unless $_[0] and $NODE_INFO{$_[0]}; $MISSING{$_[0]} = 1; } # End missing ### # nodesOfType: Returns a string for use with isi_for_array of all the nodes of the given type sub nodesOfType { return unless $_[0]; gatherProductInfo(); debug("nodesOfType: Arguments: [ $_[0] ]"); my @nodes = (); foreach my $lnn (@LNNS) { push(@nodes, $lnn) if ($NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /$_[0]/); } debug("nodesOfType: Return: [ ".join(",", @nodes)." ]"); return @nodes; } # End nodesOfType ### # parseArguments: Parses command line arguments and sets global variables. sub parseArguments { ### # Parse CLI arguments my %arguments = (); while ($_ = shift) { if (/^\d+\.\d+\.\d+\.\d+$/) { $arguments{'Target'} = $_; } elsif (/^-?-d(ebug)?$/) { $arguments{'Debug'} = 1; } elsif (/^-?-a(?:utotc)?$/) { $arguments{'AutoTC'} = 1; } elsif (/^-?-e(xtra)?$/) { $arguments{'Verbose'} = 1; } elsif (/^-?-h(elp)?$/) { $arguments{'Help'} = 1; } elsif (/^-?-j(son)?$/) { $arguments{'JSON'} = 1; } elsif (/^-?-l(?:ist)(?:=(.*))?$/) { $arguments{'CheckList'} = ($1)?$1:shift; } elsif (/^-?-r(?:un)(?:=(.*))?$/) { push(@{$arguments{'Run'}},($1)?$1:shift); } elsif (/^-?-s(imultaneous)?$/) { $arguments{'Simultaneous'} = 1; } elsif (/^-?-u(pgradeplan)?$/) { $arguments{'UpgradePlan'} = 1; } elsif (/^-?-v(ersion)?$/) { $arguments{'Version'} = 1; } elsif (/^-?-p(?:riority-)?n(?:ode-)?f(?:irmware-)?o(?:nly)?/) { $arguments{'Priority NF Only'} = 1; } elsif (/^-?-p(?:riority-)?d(?:rive-)?f(?:irmware-)?o(?:nly)?/) { $arguments{'Priority DF Only'} = 1; } elsif (/^-?-n(?:ode-)?f(?:irmware-)?(?:p(?:ackage)?)?(?:=(.*))?$/) { $arguments{'IsiFw'} = ($1)?$1:shift; } elsif (/^-?-d(?:rive-)?f(?:irmware-)?p(?:ackage)?(?:=(.*))?$/) { $arguments{'DFP'} = ($1)?$1:shift; } elsif (/^-?-d(?:rive-)?s(?:upport-)?p(?:ackage)?(?:=(.*))?$/) { $arguments{'DSP'} = ($1)?$1:shift; } elsif (/^-?-f(co)?$/) { $arguments{'F010517EE'} = 1; } elsif (/^-?-f(co-)?r(olling)?$/) { $arguments{'F010517EE'} = 1; } elsif (/^-?-f010517ee$/) { $arguments{'F010517EE'} = 1; } elsif (/^-?-f121917ee$/) { $arguments{'F121917EE'} = 1; } elsif (/^-?-f011018ee$/) { $arguments{'F011018EE'} = 1; } elsif (/^-?-t030518ee$/) { $arguments{'T030518EE'} = 1; } elsif (/^-?-f033018ee$/) { $arguments{'F033018EE'} = 1; } elsif (/^-?-f082318ee$/) { $arguments{'F082318EE'} = 1; } elsif (/^-?-t102518ee$/) { $arguments{'T102518EE'} = 1; } elsif (/^--definition$/) { $arguments{'Definition'} = 1; } else { printCheck("Isilon On-Cluster Analysis", $RPS{'Version'}); print("\nUnrecognized command line argument provided: $_\n\n"); printUsage(); exit; } } ### # Validate arguments ### our(%NodeFirmware, %DriveFirmware, %DriveSupport, %Code_Matrix, %CheckLists); # Define any undefined arguments as an empty string. foreach my $key (keys %arguments) { $arguments{$key} = "" if not defined $arguments{$key}; } # Prints the check definition if ($arguments{'Definition'}) { printDefinitionJSON(); exit; } # Enables JSON mode if ($arguments{'JSON'}) { no warnings qw(redefine); *IOCA::printResults = \&IOCA::printResultsJSON; *IOCA::printCheck = sub { }; *IOCA::printUrgent = sub { }; use warnings qw(redefine); } # Display version printCheck("Isilon On-Cluster Analysis", $RPS{'Version'}); # Exits after displaying version if ($arguments{'Version'}) { exit; } # Displays help screen and exits if ($arguments{'Help'}) { printUsage(); exit; } # Validates target OneFS is recognized if (defined $arguments{'Target'} and not grep {$_ eq $arguments{'Target'}} grep {$_ =~ /^\d+\.\d+\.\d+\.\d+$/} keys %Code_Matrix) { print("\nRequested OneFS target version, $arguments{'Target'}, was not recognized."); print("\nAvailable versions to check are: ".join(", ",sort {compareVersions($b, $a)} grep {$_ =~ /^\d+\.\d+\.\d+\.\d+$/} keys %Code_Matrix)."\n\n"); exit; } # Validates target Node Firmware Package is recognized if (defined $arguments{'IsiFw'} and not grep {$_ eq $arguments{'IsiFw'}} keys %NodeFirmware) { print("\nRequested Node Firmware Package version, $arguments{'IsiFw'}, was not recognized."); print("\nAvailable versions to check are: ".join(", ",sort {compareVersions($b, $a)} keys %NodeFirmware)."\n\n"); exit; } # Validates target Drive Firmware Package is recognized if (defined $arguments{'DFP'} and not grep {$_ eq $arguments{'DFP'}} keys %DriveFirmware) { print("\nRequested Drive Firmware Package version, $arguments{'DFP'}, was not recognized."); print("\nAvailable versions to check are: ".join(", ",sort {compareVersions($b, $a)} keys %DriveFirmware)."\n\n"); exit; } # Validates target Drive Support Package is recognized if (defined $arguments{'DSP'} and not grep {$_ eq $arguments{'DSP'}} keys %DriveSupport) { print("\nRequested Drive Support Package version, $arguments{'DSP'}, was not recognized."); print("\nAvailable versions to check are: ".join(", ",sort {compareVersions($b, $a)} keys %DriveSupport)."\n\n"); exit; } # Only allow checking for one FCO/TSE at a time if (1 < grep {defined} ($arguments{'T102518EE'}, $arguments{'T030518EE'}, $arguments{'F010517EE'}, $arguments{'F121917EE'}, $arguments{'F011018EE'}, $arguments{'F033018EE'}, $arguments{'F082318EE'})) { # Only one FCO should be specified at any given time? print("\nMultiple FCO arguments provided, only one TSE/FCO should be handled at once.\n\n"); exit; } # Validates run argument is supported if (defined $arguments{'Run'}) { my @unrecognized = (); my @checks = (); foreach my $check (@{$arguments{'Run'}}) { if (defined $check and $CHECKS{$check}) { push(@checks, $check); } else { push(@unrecognized, $check); } } if (@unrecognized) { print("\nRequested checks, ".join(", ", @unrecognized).", was not recognized."); print("\nAvailable checks are: "); foreach my $check (sort keys %CHECKS) { next unless $CHECKS{$check}{'Description'}; printf("\n\t%-30s%s", $check, $CHECKS{$check}{'Description'}); } print("\n\n"); exit; } if (@checks) { @{$arguments{'Run'}} = @checks; } else { delete $arguments{'Run'}; } } # Validates list argument is supported if (defined $arguments{'CheckList'} and not grep {$_ eq $arguments{'CheckList'}} keys %CheckLists) { print("\nRequested check list, $arguments{'CheckList'}, was not recognized."); print("\nAvailable check lists are: ".join(", ", sort keys %CheckLists)); print("\n\n"); exit; } # F010517EE if ($arguments{'F010517EE'}) { if ($arguments{'IsiFw'} and compareVersions($arguments{'IsiFw'}, "9.3.5") < 0) { print("\nRequested Node Firmware Package version, $arguments{'IsiFw'}, is less than Node Firmware Package 9.3.5 which is required for FCO F010517EE.\n\n"); exit; } if ($arguments{'Target'} and (($arguments{'Target'} =~ /^7/ and compareVersions($arguments{'Target'}, "7.2.1.6") < 0) or ($arguments{'Target'} =~ /^8/ and compareVersions($arguments{'Target'}, "8.0.0.4") < 0) or ($arguments{'Target'} =~ /^8\.0\.1\.0/))) { print("\nDestination OneFS version, $arguments{'Target'}, is less than the minimum required for FCO F010517EE: OneFS 8.0.0.4 or later, OneFS 8.0.1.1 or later, or OneFS 7.2.1.6 or later.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; } # F121917EE if ($arguments{'F121917EE'}) { if ($arguments{'IsiFw'} and compareVersions($arguments{'IsiFw'}, "10.1.2") < 0) { print("\nDestination Node Firmware Package version, $arguments{'IsiFw'}, is less than Node Firmware Package 10.1.2 which is required for FCO F121917EE.\n\n"); exit; } if ($arguments{'Target'}) { print("\nFCO F121917EE does not include a OneFS update, please re-run without a destination OneFS version.\n\n"); exit; } if ($arguments{'DSP'} or $arguments{'DFP'}) { print("\nFCO F121917EE does not include drive firmware updates, please re-run without any drive firmware versions.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; $arguments{'DSP'} = "None"; $arguments{'DFP'} = "None"; } # F011018EE if ($arguments{'F011018EE'}) { if ($arguments{'IsiFw'} and compareVersions($arguments{'IsiFw'}, "10.1.2") < 0) { print("\nDestination Node Firmware Package version, $arguments{'IsiFw'}, is less than Node Firmware Package 10.1.2 which is required for FCO F011018EE.\n\n"); exit; } if ($arguments{'Target'}) { print("\nFCO F011018EE does not include a OneFS update, please re-run without a destination OneFS version.\n\n"); exit; } if ($arguments{'DSP'} or $arguments{'DFP'}) { print("\nFCO F011018EE does not include drive firmware updates, please re-run without any drive firmware versions.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; $arguments{'DSP'} = "None"; $arguments{'DFP'} = "None"; } # T030518EE if ($arguments{'T030518EE'}) { if ($arguments{'Target'} and compareVersions($arguments{'Target'}, "8.0.0.6") < 0) { print("\nTSE T030518EE includes a OneFS update to OneFS 8.0.0.6 and later, please re-run a destination OneFS version of OneFS 8.0.0.6 or later.\n\n"); exit; } if ($arguments{'DSP'} and compareVersions($arguments{'DSP'}, "1.23") < 0) { print("\nTSE T030518EE includes drive firmware updates to Drive Support Package 1.23 and later, please re-run with a Drive Support Package of 1.23 or later.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; } # F033018EE if ($arguments{'F033018EE'}) { if ($arguments{'Target'} and compareVersions($arguments{'Target'}, "8.1.2.0") < 0) { print("\nFCO F033018EE includes a OneFS update to OneFS 8.1.2.0 and later, please re-run with a destination OneFS version of OneFS 8.1.2.0 or later.\n\n"); exit; } if ($arguments{'DSP'} and compareVersions($arguments{'DSP'}, "1.24") < 0) { print("\nFCO F033018EE includes drive firmware updates to Drive Support Package 1.24 and later, please re-run with a Drive Support Package of 1.24 or later.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; } # F082318EE if ($arguments{'F082318EE'}) { if ($arguments{'Target'} and compareVersions($arguments{'Target'}, "8.1.0.4") < 0) { print("\nFCO F082318EE includes a OneFS update to OneFS 8.1.0.4 and later, please re-run with a destination OneFS version of OneFS 8.1.0.4 or later.\n\n"); exit; } if ($arguments{'IsiFw'} and compareVersions($arguments{'IsiFw'}, "10.1.6") < 0) { print("\nFCO F082318EE includes node firmware updates to Node Firmware Package 10.1.6 and later, please re-run with a Node Firmware Package of 10.1.6 or later.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; } # T102518EE if ($arguments{'T102518EE'}) { if ($arguments{'Target'} and compareVersions($arguments{'Target'}, "8.1.0.4") < 0) { print("\nTSE T102518EE includes a OneFS update to OneFS 8.1.0.4 and later, please re-run with a destination OneFS version of OneFS 8.1.0.4 or later.\n\n"); exit; } if ($arguments{'IsiFw'} and compareVersions($arguments{'IsiFw'}, "10.2.1") < 0) { print("\nTSE T102518EE includes node firmware updates to Node Firmware Package 10.2.1 and later, please re-run with a Node Firmware Package of 10.2.1 or later.\n\n"); exit; } if ($arguments{'DSP'} and compareVersions($arguments{'DSP'}, "1.28") < 0) { print("\nTSE T102518EE includes drive firmware updates to Drive Support Package 1.28 and later, please re-run with a Drive Support Package of 1.28 or later.\n\n"); exit; } $arguments{'UpgradePlan'} = 1; } # If -u or a target OneFS code level is specified, run pre-upgrade checks if (!$arguments{'CheckList'} and ($arguments{'UpgradePlan'} or $arguments{'Target'})) { $arguments{'CheckList'} = "pre-upgrade"; } # Remove colored output for AutoTCs if ($arguments{'AutoTC'}) { no warnings 'redefine'; *colored = sub { return $_[0] if $_[0]; } } # Store arguments foreach my $key (keys %arguments) { $RPS{$key} = $arguments{$key}; } } # End parseArguments sub parseRules { # Operators: Append -> ++, Insert -> +=, Replace -> =>, Remove Groups -> --, Join -> &= # Options: [] my @rules = (); return @rules unless $_[0]; while ($_[0] =~ /\\?"?(\S+)\s+(\+\+|\+=|=>|--|&=)\s+(\S+)\s*(\[\S+\])?\\?"?/g) { my $rule = {}; $rule->{'left'} = $1; $rule->{'operator'} = $2; $rule->{'right'} = $3; $rule->{'options'} = $4 if $4; push(@rules, $rule); debug("parseRules: Left: $rule->{'left'} Operator: $rule->{'operator'} Right: $rule->{'right'}". ($rule->{'options'}?" Options: $rule->{'options'}":"")."\n"); } return @rules; } ### # printCheck: Prints header for starting checks and, if a second # argument is provided, prints that as the result of the check. sub printCheck { printf "%-50s", $_[0] if ($_[0]); printf "%s\n", $_[1] if ($_[1]); } # End printCheck ### # printDefinitionJSON: Prints the health check framework definition in JSON sub printDefinitionJSON { # Create an array of items which will be the definitions of each check my @items = (); # Add full IOCA as an item my %ioca = ( 'name' => "ioca", 'reference' => "https://support.emc.com/kb/503265", 'summary' => "Runs IOCA health checks", 'description' => "", 'freshness' => 10, 'parameters' => [], 'unsupported' => []); push(@items, \%ioca); # Add each individual check as an item foreach my $check (sort keys %CHECKS) { next unless $CHECKS{$check}{'Description'}; my %item = (); $item{'name'} = "ioca_$check"; $item{'reference'} = ""; $item{'summary'} = $CHECKS{$check}{'Description'}; $item{'description'} = ""; $item{'freshness'} = 10; @{$item{'parameters'}} = (); @{$item{'unsupported'}} = (); push(@items, \%item); } # Create the definition hash my %definition = (); $definition{'version'} = $RPS{'Version'}; $definition{'node'} = "false"; $definition{'cost'} = 5; @{$definition{'items'}} = @items; # Print the definition print toJSON(\%definition)."\n"; } ### # printResults: If provided an argument, prints it followed by an end # line, otherwise evaluates variables used in checks and prints results. sub printResults { # If there was any nodes missing information fail("Missing information for the following nodes: " .compressRange(keys %MISSING)) if (%MISSING); # Colors for each level of failure my $FAIL = colored(($_[0])?"FAIL ($_[0])":"FAIL", 'red')."\n"; my $WARN = colored(($_[0])?"WARN ($_[0])":"WARN", 'yellow')."\n"; my $PASS = colored(($_[0])?"PASS ($_[0])":"PASS", 'green')."\n"; my $INFO = colored(($_[0])?"INFO ($_[0])":"INFO", 'blue')."\n"; if (@FAIL) { print $FAIL; $RPS{'Issues'}++; } elsif (@CRITICAL) { print $FAIL; } elsif (@WARN) { print $WARN; } elsif (@INFO) { print $INFO; } else { print $PASS; } print colored(" FAIL: ", 'red')."$_\n" foreach (@FAIL); print colored(" CRITICAL: ", 'red')."$_\n" foreach (@CRITICAL); print colored(" WARN: ", 'yellow')."$_\n" foreach (@WARN); print " DETAILS: $_\n" foreach (@DETAILS); print colored(" INFO: ", 'blue')."$_\n" foreach (@INFO); print " $_\n" foreach (@DEBUG); @FAIL = (); @CRITICAL = (); @WARN = (); @INFO = (); @DETAILS = (); @DEBUG = (); %MISSING = (); } # End printResults ### # printResultsJSON: If provided an argument, prints it followed by an end # line, otherwise evaluates variables used in checks and prints results. sub printResultsJSON { # If there was any nodes missing information fail(sprintf('Missing information for the following nodes: %s', compressRange(keys %MISSING))) if %MISSING; my %result = (); my $subroutine = (caller(1))[3]; $subroutine =~ s/^IOCA:://; $result{'name'} = "ioca_$subroutine"; $result{'node'} = "cluster"; %{$result{'parameters'}} = (); $result{'status'} = "OK"; $result{'value'} = 100; $result{'status'} = "WARNING" if @WARN; $result{'value'} = 75 if @WARN; $result{'status'} = "CRITICAL" if @FAIL or @CRITICAL; $result{'value'} = 50 if @FAIL or @CRITICAL; @{$result{'details'}} = (); push(@{$result{'details'}}, "FAIL: $_") foreach (@FAIL); push(@{$result{'details'}}, "CRITICAL: $_") foreach (@CRITICAL); push(@{$result{'details'}}, "WARN: $_") foreach (@WARN); push(@{$result{'details'}}, "DETAILS: $_") foreach (@DETAILS); push(@{$result{'details'}}, "INFO: $_") foreach (@INFO); push(@{$result{'details'}}, "$_") foreach (@DEBUG); push(@RESULTS, \%result); @FAIL = (); @CRITICAL = (); @WARN = (); @INFO = (); @DETAILS = (); @DEBUG = (); %MISSING = (); } # End printResultsJSON sub printUrgent { return unless $_[0]; print colored(sprintf("## %-65s ##\n", $_[0]), "bold red"); } # printUsage: Displays help message sub printUsage { print < Checks whether updates are available in the provided node firmware package. -dsp, --drive-support-package Checks whether updates are available in the provided drive support package. -dfp, --drive-firmware-package Checks whether updates are available in the provided drive firmware package. --f010517ee Includes an upgrade plan tailored for FCO F010517EE --f121917ee Includes an upgrade plan tailored for FCO F121917EE --f011018ee Includes an upgrade plan tailored for FCO F011018EE --t030518ee Includes an upgrade plan tailored for TSE T030518EE --f033018ee Includes an upgrade plan tailored for FCO F033018EE --t102518ee Includes an upgrade plan tailored for TSE T102518EE HELPMSG } # End printUsage ### # toJSON: Converts the first argument to JSON and returns it as a string sub toJSON { return '""' unless defined $_[0]; my $ref = ref($_[0]); if (!$ref and ($_[0] eq "false" or $_[0] eq "true" or $_[0] =~ /^[0-9]+(\.[0-9]+)?$/)) { return $_[0]; } elsif (!$ref) { my $obj = $_[0]; my %esc = ( "\n" => '\n', "\r" => '\r', "\t" => '\t', "\f" => '\f', "\b" => '\b', "\"" => '\"', "\\" => '\\\\', "\'" => '\\\'', ); $obj =~ s/([\x22\x5c\n\r\t\f\b])/$esc{$1}/g; return sprintf('"%s"', $_[0]); } elsif ($ref eq "ARRAY") { return sprintf('[%s]', join(', ', map { toJSON($_) } @{$_[0]})); } elsif ($ref eq "HASH") { return sprintf('{%s}', join(', ', map { sprintf('%s: %s', toJSON($_), toJSON($_[0]{$_})) } keys %{$_[0]})); } } # End toJSON ### # upgradeDigit: Returns the digit the upgrade being performed is. # # Example: 8.1.0.4, 8.1.2.0 = 3rd digit difference sub upgradeDigit { return 4 unless @_ and scalar(@_) == 2; my @left = split(/\./, $_[0]); my @right = split(/\./, $_[1]); return 4 unless scalar(@left) == 4 and scalar(@right) == 4; for (my $i = 1; $i <= 4; $i++) { return $i if $left[$i - 1] ne $right[$i - 1]; } return 4; } # End upgradeDigit ### # verbose: Record verbose related information, but only if enabled for verbose sub verbose { push(@DEBUG, @_) if (@_ and ($RPS{'Verbose'} or $RPS{'Debug'})); return @_; } # End verbose ### # verboseHeader: A verbose header sub verboseHeader { return unless $_[0]; verbose("", "==============================", " $_[0]", "=============================="); } # End verboseHeader ### # warning: Record warn related information, but only if enabled for warn sub warning { push(@WARN, @_) if @_; return @_; } # End warning ### # wasCalled: Returns 1 if the parent subroutine has already called # wasCalled, otherwise adds subroutine to list of called subroutines. sub wasCalled { my $subroutine = (caller(1))[3]; if (scalar(grep {$_ eq $subroutine} @{$RPS{'Called'}})) { return 1; } else { push(@{$RPS{'Called'}}, $subroutine); debug("--------------------"); debug(" Called: $subroutine"); debug("--------------------"); return 0; } } # End wasCalled ### # whereRunning: returns 1 when ran on a cluster, 0 when in the root directory of a logset. sub whereRunning { # Find out if running on logs(return 0) or live(return 1) # Check to see if /ifs exists on this system if (-d "/ifs") { # /ifs exists, assuming on a Live system return 1; } elsif (-e "command_line") { # /ifs doesn't exist but command_line does, assuming we are in the root directory of a logset on elvis return 0; } printUrgent("Unable to determine where script is running, please run on a live cluster or in the root directory of a logset."); exit; } # End whereRunning $CHECKS{'checkKB516613'}{'Description'} = "Checks if any node meets criteria for KB516613"; sub checkKB516613 { return if wasCalled(); gatherProductInfo(); return unless $CLUSTER_INFO{'Gen5'}; printCheck("KB516613"); gatherNodeFirmware(); # Gather information from the cluster, not available via logs my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster (qq[ 2>&1 ${SUDO} isi_for_array -n ]. compressRange(@{$CLUSTER_INFO{'Gen5'}}). qq[ '/sbin/nvmecontrol logpage -p 3 lnv0 | grep "Slot"' ]); } # Analyze information for issues. my %nodes_with_slot_3_active = (); my %nodes_with_old_lox = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); next unless $NODE_INFO{$lnn}{'Generation'}; next unless $NODE_INFO{$lnn}{'Generation'} eq 'Gen5'; # Check LOx NVRAM card firmware version being impacted if ($NODE_INFO{$lnn}{'Firmware'}) { foreach my $device (keys %{$NODE_INFO{$lnn}{'Firmware'}}) { next unless $device =~ /LOx/; next unless $NODE_INFO{$lnn}{'Firmware'}{$device} =~ /rp180[abc]/; $nodes_with_old_lox{$lnn} = 1; } } # Gathered details required from here next unless $gather{$lnn}; # Check if Slot 3 is active with an impacted firmware version foreach my $line (@{$gather{$lnn}}) { verbose($line); next unless $line =~ /Slot 3:.*Active.*?rp180[abc]/; $nodes_with_slot_3_active{$lnn} = 1; } } # Flag the issues if (%nodes_with_old_lox) { warning("LOx NVRAM card running older rpc180[abc] firmware version on the following nodes: ".compressRange(keys %nodes_with_old_lox)); } if (%nodes_with_slot_3_active) { $RPS{'KB516613'} = 1; critical("KB 516613 criteria met on the following nodes: ".compressRange(keys %nodes_with_slot_3_active)); } if (!$RPS{'Live'} and %nodes_with_old_lox) { $RPS{'KB516613'} = 1; critical("KB 516613 criteria cannot be verified in logs, it will be assumed criteria is met."); } if ($RPS{'KB516613'}) { critical("Nodes that meet criteria for KB 516613 may fail to boot and require on-site remediation following KB 509549."); warning("Try to avoid any reboots before applying node firmware updates."); } # Output Check result info("Refer to KB516613 (https://support.emc.com/kb/516613) for details.") if (@INFO or @WARN or @FAIL); printResults(); } # End checkKB516613 $CHECKS{'checkNodeFirmware'}{'Description'} = "Checks node firmware for updates"; sub checkNodeFirmware { return if wasCalled(); # Skip firmware checks if no firmware version to check against return unless $RPS{'IsiFw'} or $RPS{'IsiFw'} ne "None"; # Skip firmware checks if the cluster is virtual gatherProductInfo(); return if $CLUSTER_INFO{'Virtual'}; checkPatches(); checkPartitions(); checkKB521890(); checkKB516613(); checkKB519119(); our %NodeFirmware; # Flag that node firmware updates will not be accurate due to a BMC firmware package being installed. fail("Node firmware updates will not be accurate when any BMC firmware package is installed. Uninstall the BMC firmware package and re-run health checks.") if hasPatch("IsiFw.*BMC"); # Revert to 9.3.5 if running a OneFS version prior to 7.1 and not upgrading to a OneFS version of 7.1 or greater. if (compareVersions($RPS{'IsiFw'}, "9.3.5") > 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") < 0 and (!$RPS{'Target'} or compareVersions($RPS{'Target'}, "7.1") < 0)) { details("Node Firmware Package $RPS{'IsiFw'} is not supported on OneFS versions prior to 7.1, Node Firmware Package 9.3.5 must be used and will be used in checks."); $RPS{'IsiFw'} = "9.3.5"; } # Recommend 9.3.5 if no Gen5/6 nodes. This may change if some point in the future newer updates become available for older generation hardware if (compareVersions($RPS{'IsiFw'}, "9.3.5") > 0 and !$CLUSTER_INFO{'Gen6'} and !$CLUSTER_INFO{'Gen5'} and !$CLUSTER_INFO{'A100'}) { $RPS{'IsiFw'} = "9.3.5"; details("This cluster contains no Gen5, Gen6, or A100 nodes. Node Firmware Package 9.3.5 will be selected as it contains the latest updates for Gen4 and older hardware."); } printCheck("Node Firmware ($RPS{'IsiFw'})"); gatherNodeFirmware(); gatherQLogicFirmware(); gatherLOxFirmware(); gatherNetworkInterfaces(); # Analyze information for issues. my $power_supply_firmware_issue = 0; my $nvram_lox_issue = 0; my $nvram_lox_issue2 = 0; my @nvram_lox_issue3 = (); my $requires_bxe_patch = 0; my @fwpkgs = sort {compareVersions($b, $a)} keys %NodeFirmware; foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'Firmware'}) { missing($lnn); next; } my $firmware_out_of_date = 0; DEVICE: foreach my $device (keys %{$NODE_INFO{$lnn}{'Firmware'}}) { next unless $device and $NODE_INFO{$lnn}{'Firmware'}{$device}; my $firmware = $NODE_INFO{$lnn}{'Firmware'}{$device}; debug("checkNodeFirmware: Node $lnn Device: $device Firmware: $firmware"); # Check for an issue where the LOx reports unknown information if ($device =~ /unknown:LOx/i and ! grep(/^lox/i, keys %{$NODE_INFO{$lnn}{'Firmware'}}) and compareVersions($CLUSTER_INFO{'OneFS'}, "7.2.1") < 0) { push(@nvram_lox_issue3, $lnn); debug("checkNodeFirmware: !!! Node $lnn is reporting unknown:LOx information"); } # If the firmware version is found in the list it is up to date. PACKAGE: foreach my $fwpkg (@fwpkgs) { # Stop looking if we get to a package that is less than what we were asked to check for last PACKAGE if compareVersions($fwpkg, $RPS{'IsiFw'}) < 0; # Firmware updates not available in this package for this device next PACKAGE unless $NodeFirmware{$fwpkg}{$device}; # Current firmware version is reported in this package so already up to date next DEVICE if (grep {$firmware eq $_} @{$NodeFirmware{$fwpkg}{$device}}); } # Check for an issue where a node is reporting multiple types of Power Supply firmware versions if (scalar(grep(/^CFFPS/, keys %{$NODE_INFO{$lnn}{'Firmware'}})) > 2) { debug("checkNodeFirmware: !!! Multiple incorrectly reported Power Supply Firmware detected"); foreach my $tmp (grep(/^CFFPS/, keys %{$NodeFirmware{$RPS{'IsiFw'}}})) { next DEVICE if (grep {$firmware eq $_} @{$NodeFirmware{$RPS{'IsiFw'}}{$tmp}}); } } # Updates for this device not available in the selected firmware package next DEVICE unless $NodeFirmware{$RPS{'IsiFw'}}{$device}; # Firmware is out of date $firmware_out_of_date = 1; debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware out of date, valid recent firmware revisions: ".join(",", @{$NodeFirmware{$RPS{'IsiFw'}}{$device}})); push(@{$RPS{'Node Firmware'}{'Devices'}{$device}}, $lnn); # Check for ETA 184813 if ($device eq "Lsi" and $firmware =~ /^[\d\.]+$/ and compareVersions($firmware, "10.00.06.00") <= 0) { push(@{$RPS{'ETA'}{'184813'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by ETA 184813"); } # Check for ETA 184406 if ($device eq "Mellanox" and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^A100/ and $firmware =~ /^([\d\.]+)/ and compareVersions($1, "2.30.8020") < 0) { push(@{$RPS{'ETA'}{'184406'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by ETA 184406"); } # Check for KB466373/FCO F010517EE if ($device =~ "^CMC_" and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(S210|X210|X410|NL410|HD400)/ and compareVersions($firmware, "02.05") < 0) { push(@{$RPS{'FCO'}{'F010517EE'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F010517EE"); } # Check for KB513206/FCO F121917EE if ($device eq "EPpost_warthog" and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(H500|H600|F800)/ and compareVersions($firmware, "20.81") < 0) { push(@{$RPS{'FCO'}{'F121917EE'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F121917EE"); } # Check for KB515633/FCO F011018EE if ($device eq "EPsas_infinity" and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^F800/ and compareVersions($firmware, "0001.0004.0000.11181") < 0) { push(@{$RPS{'FCO'}{'F011018EE'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F011018EE"); } # Check for FCO F082318EE, checks that all components with updates available in 10.1.6 are running at least 10.1.6 versions if (($device eq "DEbcc_infinity" and compareVersions($firmware, "00.71") < 0) or ($device eq "DEbios_banshee" and compareVersions($firmware, "33.18") < 0) or ($device eq "DEbmc_main" and compareVersions($firmware, "22.71") < 0) or ($device eq "DEfp_fpinffw" and compareVersions($firmware, "02.0c") < 0) or ($device eq "DEpost_banshee" and compareVersions($firmware, "26.86") < 0) or ($device eq "DEps0_gen2_artesyn" and compareVersions($firmware, "02.14") < 0) or ($device eq "DEps1_gen2_artesyn" and compareVersions($firmware, "02.14") < 0) or ($device eq "EPbcc_infinity" and compareVersions($firmware, "00.71") < 0) or ($device eq "EPbmc_main" and compareVersions($firmware, "22.71") < 0) or ($device eq "EPfp_fpinffw" and compareVersions($firmware, "02.0c") < 0) or ($device eq "EPpost_warthog" and compareVersions($firmware, "20.86") < 0) or ($device eq "EPps0_gen2_artesyn" and compareVersions($firmware, "02.14") < 0) or ($device eq "EPps1_gen2_artesyn" and compareVersions($firmware, "02.14") < 0)) { debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F011018EE"); #@{$RPS{'FCO'}{'F082318EE'}{'Nodes'}} = merge(@{$RPS{'FCO'}{'F082318EE'}{'Nodes'}}, $lnn); } # Check for known issue with S210 Power Supply firmware updates on OneFS versions prior to 7.1.1.5. if ($device =~ /Octane/ and $firmware =~ /8\.02/ and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1.5") < 0) { $power_supply_firmware_issue = 1; debug("checkNodeFirmware: !!! Known issue with S210 nodes and power supply firmware updates on OneFS versions 7.1.1.[01234]"); } if ($device =~ /LOx/ and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1.9") < 0) { $nvram_lox_issue = 1; debug("checkNodeFirmware: !!! Known issue with LOx and OneFS 7.1.1.[012345678] branch: https://support.emc.com/kb/465882"); } if ($device =~ /LOx/ and !($firmware =~ /\+/)) { $nvram_lox_issue2 = 1; debug("checkNodeFirmware: !!! Known issue with LOx and OneFS 7.2.0.[0123] and OneFS 7.1.1.[012345] where secondary firmware does not show or get updated in these versions"); } # A flag if QLogic firmware updates are needed if ($device =~ /QLogic-NX2/ and compareVersions($firmware, "7.13.0") < 0) { $RPS{'Node Firmware'}{'QLogic'} = 1; } } push(@{$RPS{'Node Firmware'}{'Nodes'}}, $lnn) if $firmware_out_of_date; } # NVRam LOx known issue should be displayed regardless of updates being available if (@nvram_lox_issue3) { info("Unable to determine if NVRam LOx firmware updates are needed due to it reporting ."); info("If NVRam LOx updates are needed, you must first upgrade the OneFS version of the cluster to 7.1.1.9 or above.") if compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1.8") <= 0; } # Node Firmware updates available if ($RPS{'Node Firmware'}{'Nodes'}) { # Show nodes requiring updates info(scalar(@{$RPS{'Node Firmware'}{'Nodes'}})." nodes have out of date node firmware: ".compressRange(@{$RPS{'Node Firmware'}{'Nodes'}})); debug("checkNodeFirmware: !!! Devices with out of date firmware: ".join(", ", keys %{$RPS{'Node Firmware'}{'Devices'}})); # Display ETAs and known issues if ($RPS{'ETA'}{'184406'}{'Nodes'}) { $RPS{'ETA'}{'184406'}{'Severity'} = "INFO"; $RPS{'ETA'}{'184406'}{'Message'} = scalar(@{$RPS{'ETA'}{'184406'}{'Nodes'}})." nodes are running firmware versions affected by ETA 184406: ".compressRange(@{$RPS{'ETA'}{'184406'}{'Nodes'}}); warning($RPS{'ETA'}{'184406'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'ETA'}{'184406'}{'Message'}"); } if ($RPS{'ETA'}{'184813'}{'Nodes'}) { $RPS{'ETA'}{'184813'}{'Severity'} = "INFO"; $RPS{'ETA'}{'184813'}{'Message'} = scalar(@{$RPS{'ETA'}{'184813'}{'Nodes'}})." nodes are running firmware versions affected by ETA 184813: ".compressRange(@{$RPS{'ETA'}{'184813'}{'Nodes'}}); warning($RPS{'ETA'}{'184813'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'ETA'}{'184813'}{'Message'}"); } if ($RPS{'FCO'}{'F010517EE'}{'Nodes'}) { $RPS{'FCO'}{'F010517EE'}{'Severity'} = "INFO"; $RPS{'FCO'}{'F010517EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F010517EE'}{'Nodes'}})." nodes are running firmware versions affected by FCO F010517EE: ".compressRange(@{$RPS{'FCO'}{'F010517EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F010517EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F010517EE'}{'Message'}"); } if ($RPS{'FCO'}{'F121917EE'}{'Nodes'}) { $RPS{'FCO'}{'F121917EE'}{'Severity'} = "INFO"; $RPS{'FCO'}{'F121917EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F121917EE'}{'Nodes'}})." nodes are running firmware versions affected by FCO F121917EE: ".compressRange(@{$RPS{'FCO'}{'F121917EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F121917EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F121917EE'}{'Message'}"); } if ($RPS{'FCO'}{'F011018EE'}{'Nodes'}) { $RPS{'FCO'}{'F011018EE'}{'Severity'} = "INFO"; $RPS{'FCO'}{'F011018EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F011018EE'}{'Nodes'}})." nodes are running firmware versions affected by FCO F011018EE: ".compressRange(@{$RPS{'FCO'}{'F011018EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F011018EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F011018EE'}{'Message'}"); } if ($RPS{'FCO'}{'F082318EE'}{'Nodes'}) { $RPS{'FCO'}{'F082318EE'}{'Severity'} = "INFO"; $RPS{'FCO'}{'F082318EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F082318EE'}{'Nodes'}})." nodes are running node firmware versions affected by FCO F082318EE: ".compressRange(@{$RPS{'FCO'}{'F082318EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F082318EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F082318EE'}{'Message'}"); } my @patch_211361_nodes = nodesOfType("^(H500|H600|F800)"); if (not hasPatch('patch-211361') and $CLUSTER_INFO{'OneFS'} =~ /^8\.1\.0\.[01]$/ and @patch_211361_nodes) { $RPS{'Node Firmware'}{'Patch'}{'patch-211361'}{'Impacts'} = ""; if ($RPS{'F121917EE'} or $RPS{'F011018EE'} or $RPS{'FCO'}{'F121917EE'}{'Nodes'}) { my $msg = "Certain voltage threshold monitoring values were incorrectly programmed to be too stringent for H500, H600, and F800 nodes. This could result in unnecessary preventative action by the system, such as powering down. This issue is resolved in Node Firmware Package 10.1.1 and later, but before you perform a node firmware update on F800, H600, or H500 nodes, you must install Patch 211361."; push(@{$RPS{'Node Firmware'}{'Priority'}}, $msg); warning($msg); } warning("If performing node firmware updates to H500, H600, and/or F800 nodes on OneFS $CLUSTER_INFO{'OneFS'}, patch 211361 must first be installed."); } if ($power_supply_firmware_issue) { info("There is a known issue that power supply firmware updates for S210 nodes may fail to update from 8.02 to 9.05 when running OneFS 7.1.1.0 to 7.1.1.4. For the power supply firmware to be updated, you must upgrade the OneFS version of the cluster to 7.1.1.5 or above."); } if ($nvram_lox_issue) { info("There is a known issue that NVRam LOx updates fail when running OneFS 7.1.1.0 to 7.1.1.8. For the NVRam to be updated, you must upgrade the OneFS version of the cluster to 7.1.1.9 or above."); } if ($nvram_lox_issue2) { info("There is a known issue that NVRam LOx will not display or update its secondary component when running OneFS 7.1.1.0 to 7.1.1.5 or 7.2.0.0 to 7.2.0.3. For the NVRam secondary component to be displayed and updated, you must first upgrade the OneFS version of the cluster to a version not affected."); } if ($RPS{'Node Firmware'}{'QLogic'} and $CLUSTER_INFO{'OneFS'} =~ /^7\.1\.1\.([0123456789]|10|11)$|^7\.2\.0\.[012345]$|^7\.2\.1\.[012345]$|^8\.0\.0\.[01234]/) { $requires_bxe_patch = 1; info("If performing updates to QLogic firmware on OneFS $CLUSTER_INFO{'OneFS'}, a BXE patch must first be installed and all nodes with BXE network interfaces rebooted."); } if ($RPS{'Node Firmware'}{'var'} and compareVersions($RPS{'IsiFw'}, "10.0.1") >= 0) { fail("To install Node Firmware Package $RPS{'IsiFw'}, a minimum of 500MB of free space is required in /var, the following nodes do not meet this requirement: ".compressRange(@{$RPS{'Node Firmware'}{'var'}})); } # Add the Gen6 Power Supply Check Script to files to download push(@{$RPS{'Node Firmware'}{'Files'}}, "Gen6 Power Supply Check Script") if $CLUSTER_INFO{'Gen6'}; # Add the package to required files push(@{$RPS{'Node Firmware'}{'Files'}}, "Isilon Node Firmware Package $RPS{'IsiFw'}"); # Add the target version for the node firmware updates $RPS{'Node Firmware'}{'Target'} = "Node Firmware $RPS{'IsiFw'}"; # Adjust order based on current OneFS version and known issues $RPS{'Node Firmware'}{'Position'} = 1; $RPS{'Node Firmware'}{'Position'} = 2 if (compareVersions($CLUSTER_INFO{'OneFS'}, "6.5.2.85") < 0); $RPS{'Node Firmware'}{'Position'} = 2 if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") < 0 and compareVersions($RPS{'IsiFw'}, "10.0.1") >= 0); $RPS{'Node Firmware'}{'Position'} = 2 if ($power_supply_firmware_issue and $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.1.1.5") >= 0); $RPS{'Node Firmware'}{'Position'} = 2 if ($nvram_lox_issue and $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.1.1.9") >= 0); $RPS{'Node Firmware'}{'Position'} = 2 if ($nvram_lox_issue2 and $RPS{'Target'} and $RPS{'Target'} !~ /^7\.1\.1\.[012345]|^7\.2\.0\.[0123]/); $RPS{'Node Firmware'}{'Position'} = 2 if (@nvram_lox_issue3 and $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.1.1.9") >= 0); $RPS{'Node Firmware'}{'Position'} = 2 if $requires_bxe_patch; $RPS{'Node Firmware'}{'Position'} = 2 if !$RPS{'Target'}; $RPS{'Node Firmware'}{'Position'} = 2 if $RPS{'KB521890'}; $RPS{'Node Firmware'}{'Position'} = 2 if $RPS{'KB519890'}; $RPS{'Node Firmware'}{'Position'} = 2 if $RPS{'Target'} and !$RPS{'KB516613'} and compareVersions($RPS{'Target'}, "8.1.0.4") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.4") < 0; if ($RPS{'Node Firmware'}{'Position'} == 2 and $RPS{'Node Firmware'}{'Patch'} and $RPS{'Node Firmware'}{'Patch'}{'patch-211361'} and $RPS{'Target'} and compareVersions($RPS{'Target'}, "8.1.0.2") >= 0) { delete $RPS{'Node Firmware'}{'Patch'}; } if ($RPS{'F011018EE'}) { push(@{$RPS{'Node Firmware'}{'Additional Comments'}}, "Start IntegrityScan job after node firmware updates"); } # Remove Node Firmware Updates if requested only priority updates and there are no priority updates needed. if ($RPS{'Priority NF Only'} and !$RPS{'Node Firmware'}{'Priority'}) { delete $RPS{'Node Firmware'}; } } # Output Check result info("Refer to KB210513 (https://support.emc.com/kb/210513) for details.") if (@INFO or @WARN or @FAIL); printResults(); } # End checkNodeFirmware ### # gatherLOxFirmware: Gathers ( or attempts to gather ) LOx NVRAM firmware revision information. # Currently only works on a live cluster, only designed to address an issue in the 7.1.1 branch. sub gatherLOxFirmware { return if wasCalled(); return unless $RPS{'Live'}; return unless isCurrentOneFS(qr/^7\.1\.1\./); # Gather information from the cluster my %gather = (); %gather = gatherCluster("2>&1 ${SUDO} isi_for_array '/sbin/nvmecontrol identify lnv0;/sbin/nvmecontrol devinfo lnv0'"); # Look for firmware versions and assign based on found information foreach my $lnn (@LNNS) { next unless $gather{$lnn}; # If LOx was already gathered, skip to the next node. next if $NODE_INFO{$lnn}{'Firmware'}{'LOx'} or $NODE_INFO{$lnn}{'Firmware'}{'LOx2'} or $NODE_INFO{$lnn}{'Firmware'}{'LOx2-MLC-YD'}; # Reset variables my $primary = ""; my $secondary = ""; my $model = ""; # Gather information for the node foreach my $line (@{$gather{$lnn}}) { if ($line =~ /Firmware Version:\s+(\S+)/) { $primary = $1; debug("gatherLOxFirmware: Primary Firmware Version found: $primary"); } elsif ($line =~ /Secondary Boot Loader Rev:\s+(\S+)/) { $secondary = $1; debug("gatherLOxFirmware: Secondary Firmware Version found: $secondary"); } elsif ($line =~ /Model Number:\s+(\S+)/) { $model = $1; debug("gatherLOxFirmware: Model found: $model"); } } # A minimum of the model and the primary firmware version is required next unless $model and $primary; # Assign device by the model my $device = ""; if ($model =~ /303-409-000A/) { $device = "LOx2"; } elsif ($model =~ /303-409-001[AB]/) { $device = "LOx2-MLC-YD"; } else { $device = "LOx"; } debug("gatherLOxFirmware: Model identified as device: $device"); # Assign firmware information $NODE_INFO{$lnn}{'Firmware'}{$device} = "$primary".($secondary?"+$secondary":""); debug("NODE_INFO->${lnn}->Firmware->$device=$NODE_INFO{$lnn}{'Firmware'}{$device}"); # Undefine any unknown LOx device information found my @unknown_devices = grep(/unknown:LOx/, keys %{$NODE_INFO{$lnn}{'Firmware'}}); foreach my $unknown_device (@unknown_devices) { delete $NODE_INFO{$lnn}{'Firmware'}{$unknown_device}; } } } # End gatherLOxFirmware ### # gatherNodeFirmware: Gathers node firmware sub gatherNodeFirmware { return if wasCalled(); # Gathers from /etc/ifs/firmware_versions, this is information that is # updated during node reboots and /may/ be inaccurate! my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster (qq[2>&1 ${SUDO} isi_for_array cat /etc/ifs/firmware_versions]); } else { %gather = gatherCluster (qq[2>&1 grep -H . */etcifs.tar/ifs/firmware_versions]); } foreach my $lnn (@LNNS) { next unless $gather{$lnn}; foreach my $line (@{$gather{$lnn}}) { # Ignore informational lines next if $line =~ /^\s*(Device|NOTE|WARNING|-)/; # Ignore lines with unknown # next if $line =~ /unknown/; # Split by whitespace, removing groups of just whitespace my @columns = grep { /\S/ } split(/\s+/, $line); next unless @columns; # Remove the column if it is a * indicating out of date firmware shift @columns if $columns[0] eq "*"; # Must be at least four columns next unless @columns and scalar @columns >= 4; # Assign out firmware information my $device = $columns[0]; my $type = $columns[1]; my $firmware = $columns[2]; $NODE_INFO{$lnn}{'Firmware'}{$device} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${device}=$firmware"); } } # Gathers from isi firmware status, even in newer OneFS versions! If # information is found here it replaces previously found info. my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") >= 0) { @gather = gatherLocal("2>&1 ${SUDO} isi_classic firmware status"); } else { @gather = gatherLocal("2>&1 ${SUDO} isi firmware status"); } } else { @gather = gatherLocal("2>&1 cat firmware"); } foreach my $line (@gather) { # Ignore informational lines next if $line =~ /^\s*(Device|NOTE|WARNING|-)/; # Ignore unknown firmware/devices # next if $line =~ /unknown/; # Split by whitespace, removing groups of just whitespace my @columns = grep { /\S/ } split(/\s+/, $line); next unless @columns; # Remove the column if it is a * indicating out of date firmware shift @columns if $columns[0] eq "*"; # Must be at least four columns next unless @columns and scalar @columns >= 4; # The nodes column must only consist of nodes next unless $columns[$#columns] =~ /^[0-9,-]+$/; # Assign out firmware information my $device = $columns[0]; my $type = $columns[1]; my $firmware = $columns[2]; my @nodes = expandRange($columns[$#columns]); foreach my $lnn (@nodes) { # Make sure lnn is a valid node next unless $NODE_INFO{$lnn}; $NODE_INFO{$lnn}{'Firmware'}{$device} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${device}=$firmware"); if ($type eq 'CMC' and not $NODE_INFO{$lnn}{'Firmware'}{'CMC'}) { $NODE_INFO{$lnn}{'Firmware'}{'CMC'} = $firmware; debug("NODE_INFO->${lnn}->Firmware->CMC=$firmware"); } } } } # gatherNodeFirmware ### # gatherQLogicFirmware: Gathers ( or attempts to gather ) QLogic BXE firmware revision information. # This is will work 100% of the time live on a cluster but only works on logsets when the log gather # was performed with an FRU/CTO package of 201506 or newer.. sub gatherQLogicFirmware { return if wasCalled(); # Gather information from the cluster my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array sysctl dev.bxe.0.bc_version"); } else { %gather = gatherCluster("2>&1 grep -H . */asbuilt.xml"); } # Look for firmware versions and assign based on found information foreach my $lnn (@LNNS) { # Skip if gather has nothing next unless $gather{$lnn}; # Skip if already gathered next if $NODE_INFO{$lnn}{'Firmware'}{'QLogic-NX2'}; # Parse output foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /dev\.bxe\.0\.bc_version:\s+(\S+)$/ or ($line =~ /BCM57810/ and $line =~ /firmware=\"(.*?)\"/); $NODE_INFO{$lnn}{'Firmware'}{'QLogic-NX2'} = $1; debug("NODE_INFO->${lnn}->Firmware->QLogic-NX2=$1"); } } } # End gatherQLogicFirmware ### # checkAccessZones: A very dreaded failure, flags if multiple access # zones are configured for upgrades across 7.1.1. $CHECKS{'checkAccessZones'}{'Description'} = "Checks if multiple access zones are configured for upgrades across 7.1.1. Checks for nested or overlapping SMB shares. Adds warnings when any non-System access zone pools are found, only applies for upgrades across OneFS 7.1.1"; sub checkAccessZones { return if wasCalled(); # Skip if OneFS currently less than 7.0 if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.0") < 0) { debug("checkAccessZones: Current OneFS version less than 7.0, skipping."); return; } printCheck("Access Zones"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi auth local list"); } elsif (-e "local/isi_auth_local_list") { @gather = gatherLocal("2>&1 cat local/isi_auth_local_list"); } elsif (-e "local/isi_auth") { @gather = gatherLocal("2>&1 cat local/isi_auth"); } # Analyze information for issues. my $parse_total = $RPS{'Live'}; foreach my $line (@gather) { debug("checkAccessZones: $line"); $parse_total = 1 if $line =~ /auth local list$/; if ($parse_total and $line =~ /^\s*Total:\s*(\d*)$/) { $CLUSTER_INFO{'Access Zones'} = $1; debug("checkAccessZones: Total: $CLUSTER_INFO{'Access Zones'}"); last; } } if (!$CLUSTER_INFO{'Access Zones'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") < 0 and compareVersions($RPS{'Target'}, "7.1.1") >= 0) { warning("Unable to verify the number of Access Zones, manually check if there is more than just the System Access Zone configured"); } } elsif ($CLUSTER_INFO{'Access Zones'} > 1 and $RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") < 0 and compareVersions($RPS{'Target'}, "7.1.1") >= 0) { checkOverlap(); checkNonSystemPools(); warning("We strongly recommend that you read the OneFS Upgrade Planning and Process Guide and contact EMC Isilon Technical Support before upgrading the cluster to OneFS 7.1.1 or later. This will help to ensure that changes to access zones will not interrupt your workflow."); addCustomerInformation("Access Zones", @WARN, @FAIL); } info("Refer to KB210521 (https://support.emc.com/kb/210521) for details.") if (@WARN or @FAIL); # Output Check result if ($CLUSTER_INFO{'Access Zones'}) { printResults($CLUSTER_INFO{'Access Zones'}); } else { printResults(); } } # End checkAccessZones $CHECKS{'checkDestinationOneFS'}{'Description'} = "Checks destination OneFS version"; ### # Perform steps to 'adjust' the target version and verify it is a valid target sub checkDestinationOneFS { return if wasCalled(); if ($RPS{'Target'}) { if (compareVersions($RPS{'Target'}, $CLUSTER_INFO{'OneFS'}) <= 0) { # Remove the destination OneFS version if it is less than or equal # to the current OneFS version warning("Destination OneFS version $RPS{'Target'} is less than or ". "equal to the current OneFS version, removing destination ". "OneFS version."); delete $RPS{'Target'}; printCheck("Destination OneFS Version"); printResults(); } elsif (compareVersions($RPS{'Target'}, "7.1") <= 0) { info("The destination version of OneFS $RPS{'Target'} is ". "End-of-Service-Life."); info("For details see: https://support.emc.com/docu45445_Isilon-Product-Availability.pdf"); printCheck("Destination OneFS Version"); printResults(); # } elsif (compareVersions($RPS{'Target'}, "7.2") <= 0) { # info("The destination version of OneFS $RPS{'Target'} will reach ". # "End-of-Service-Life on 8/31/2019."); # info("For details see: https://support.emc.com/docu45445_Isilon-Product-Availability.pdf"); # printCheck("Destination OneFS Version"); # printResults(); } } # FCO F010517EE and not already at an FCO required OneFS version if ($RPS{'F010517EE'} and !$RPS{'Target'} and (($CLUSTER_INFO{'OneFS'} =~ /^7/ and compareVersions($CLUSTER_INFO{'OneFS'}, "7.2.1.6") < 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") < 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8\.0\.1\.0/))) { # No destination OneFS version provided if ($CLUSTER_INFO{'OneFS'} =~ /8\.0\.1\.0/) { $RPS{'Target'} = "8.1.2.0"; } else { $RPS{'Target'} = "8.0.0.7"; } warning("No Destination OneFS version provided for FCO F010517EE, ". "setting destination to recommended, OneFS $RPS{'Target'}."); printCheck("Destination OneFS Version"); printResults(); } # TSE T030518EE but no selected target and running OneFS 8.0.0.6 or lower if ($RPS{'T030518EE'} and !$RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.6") < 0) { $RPS{'Target'} = "8.0.0.7"; warning("No Destination OneFS version provided for TSE T030518EE, ". "setting destination to recommended, OneFS $RPS{'Target'}."); printCheck("Destination OneFS Version"); printResults(); } # FCO F033018EE but no selected target and running OneFS 8.1.2.0 or lower if ($RPS{'F033018EE'} and !$RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.2.0") < 0) { $RPS{'Target'} = "8.1.2.0"; warning("No Destination OneFS version provided for FCO F033018EE, ". "setting destination to recommended, OneFS $RPS{'Target'}."); printCheck("Destination OneFS Version"); printResults(); } # FCO F082318EE but no selected target and running OneFS 8.1.0.4 or lower if ($RPS{'F082318EE'} and !$RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.4") <= 0) { $RPS{'Target'} = "8.1.2.0"; warning("No Destination OneFS version provided for FCO F082318EE, ". "setting destination to recommended, OneFS $RPS{'Target'}."); printCheck("Destination OneFS Version"); printResults(); } # TSE T102518EE but no selected target and running OneFS 8.1.0.4 or lower if ($RPS{'T102518EE'} and !$RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.4") <= 0) { $RPS{'Target'} = "8.1.2.0"; warning("No Destination OneFS version provided for TSE T102518EE, ". "setting destination to recommended, OneFS $RPS{'Target'}."); printCheck("Destination OneFS Version"); printResults(); } # TSE T102518EE but OneFS 8.1.0.4 selected as destination OneFS if ($RPS{'T102518EE'} and $RPS{'Target'} and compareVersions($RPS{'Target'}, "8.1.0.4") <= 0) { warning("Destination OneFS version $RPS{'Target'} meets TSE requirements for TSE T102518EE, but OneFS 8.1.2.0 and later is recommended."); printCheck("Destination OneFS Version"); printResults(); } # Display destination OneFS version printCheck("Destination OneFS Version", $RPS{'Target'}) if $RPS{'Target'}; } # End checkDestinationOneFS $CHECKS{'checkDiskpools'}{'Description'} = 'Checks diskpools and class equivalence for OneFS upgrades going across 7.0.'; sub checkDiskpools { return if wasCalled(); # Only check for upgrades from pre to 7.0 to post 7.0. if (!$CLUSTER_INFO{'OneFS'} or !$RPS{'Target'} or compareVersions($CLUSTER_INFO{'OneFS'}, "7") >= 0 or compareVersions($RPS{'Target'}, "7") < 0) { debug("checkDiskpools: Check skipped, only applies when crossing OneFS 7.0"); return; } printCheck("Diskpools"); gatherIsiStatPools(); gatherProductInfo(); # Populate an equivalence class stored in NODE_INFO{$node}{'Class'} # based on node product information in NODE_INFO{$node}{'Product'} # Class 0 means there is no equivalence class. foreach my $node (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$node}{'Product'}; $_ = $NODE_INFO{$node}{'Product'}; if (/IQ 6000x/i or /^X200.*6144MB.*6144GB$/i) { $NODE_INFO{$node}{'Class'} = 1; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 12000x/i or /^X200.*6144MB.*12TB$/i) { $NODE_INFO{$node}{'Class'} = 2; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 36000x/i or /^X400.*24GB.*36TB$/i) { $NODE_INFO{$node}{'Class'} = 3; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 72000x/i or /^X400.*24GB.*72TB$/i) { $NODE_INFO{$node}{'Class'} = 4; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 32000X-SSD/i or /^X400.*24GB.*-32TB.*SSD$/i) { $NODE_INFO{$node}{'Class'} = 5; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 36NL/i or /^NL400.*12GB.*-36TB$/i) { $NODE_INFO{$node}{'Class'} = 6; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 72NL/i or /^NL400.*12GB.*-72TB$/i) { $NODE_INFO{$node}{'Class'} = 7; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 108NL/i or /^NL400.*12GB.*-108TB$/i) { $NODE_INFO{$node}{'Class'} = 8; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } else { $NODE_INFO{$node}{'Class'} = 0; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } } # Iterate through and compare equivalence classes for each node # using NODE_INFO{$node}{'Pool'} and NODE_INFO{$node}{'Class'} foreach my $nodea (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$nodea}{'Pool'}; next unless $NODE_INFO{$nodea}{'Class'}; foreach my $nodeb (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$nodeb}{'Class'}; next unless $NODE_INFO{$nodeb}{'Pool'}; # Move to the next node if we've already done the comparison. # I.E.: Node A compares against every node greater than it on it's # first path. So if the node b is less than the node a, it's already # been analyzed. next if (compareVersions($nodea, $nodeb) >= 0); # Node a and node b are in equivalent classes, but they are not in the same diskpool. if ($NODE_INFO{$nodea}{'Class'} and $NODE_INFO{$nodeb}{'Class'} and $NODE_INFO{$nodea}{'Class'} == $NODE_INFO{$nodeb}{'Class'} and $NODE_INFO{$nodea}{'Pool'} ne $NODE_INFO{$nodeb}{'Pool'}) { fail("Nodes $nodea and $nodeb are of equivalent type, and need to be in the same diskpool"); } # Node a and node b are in the same diskpool, and either # they are not in the same equivalence class or they are different products. if ($NODE_INFO{$nodea}{'Pool'} eq $NODE_INFO{$nodeb}{'Pool'} and ($NODE_INFO{$nodea}{'Class'} != $NODE_INFO{$nodeb}{'Class'} or (!$NODE_INFO{$nodea}{'Class'} and !$NODE_INFO{$nodeb}{'Class'} and $NODE_INFO{$nodea}{'Product'} ne $NODE_INFO{$nodeb}{'Product'}))) { fail("Nodes $nodea and $nodeb are not of equivalent type, and need to be in different diskpools"); } } } # Output Check result printResults(); } # End checkDiskPools ### # checkFilepoolPolicies: Checks filepools for final match being set and # names starting with a number. $CHECKS{'checkFilepoolPolicies'}{'Description'} = "Checks GNA requirements and checks filepools for final match being set and names starting with a number."; sub checkFilepoolPolicies { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") >= 0) { debug("checkFilepoolPolicies: Skipping. Current version greater than 7.1.1"); return; } elsif (!$RPS{'Target'}) { debug("checkFilepoolPolicies: Skipping. No target version"); return; } printCheck("Filepool Policies"); gatherForceSpill(); # Gather information debug("checkFilepoolPolicies: Gathering filepool policy information"); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig smartpools.filepools]); } else { @gather = gatherLocal(qq[2>&1 grep -h 'smartpools.filepools' local/isi_gconfig local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]); } # Analyze for issues my $policy_name = ""; my $gna_enabled = 0; my @policies_with_final_match = (); my @policies_start_with_number = (); debug("checkFilepoolPolicies: Searching for filepool policy related issues"); foreach (@gather) { if (/smartpools\.filepools\.policies\.(\d+) /i) { $policy_name = ""; } elsif (/smartpools\.filepools\.policies\.\d\.name .*\"(.*)\"/i) { $policy_name = $1; if ($policy_name =~ /^\d/) { unless (grep {$_ eq $policy_name} @policies_start_with_number) { push(@policies_start_with_number, $policy_name); debug("checkFilepoolPolicies: !!! Policy name begins with a number: $policy_name"); } } } elsif (/smartpools\.filepools.*final_match.*false/i) { unless (grep {$_ eq $policy_name} @policies_with_final_match) { push(@policies_with_final_match, $policy_name); debug("checkFilepoolPolicies: !!! Policy with final match set to false: $policy_name"); } } elsif (/smartpools\.filepools\.gna_enabled.*true/i) { debug("checkFilepoolPolicies: Found GNA is enabled"); $gna_enabled = 1; } } # Policies start with a number if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") < 0 and compareVersions($RPS{'Target'}, "7.1.1") >= 0 and @policies_start_with_number) { fail("Upgrade would invalidate all file pool policies with names that start with a number. As of OneFS 7.1.1, filepool policy names can no longer have a numeric first character. The following policies must be renamed before upgrade: ".join(",",@policies_start_with_number)); fail("Upgrade would invalidate a file pool policy that doesn't stop matching. As of OneFS 7.1.1, filepool policy matching terminates after a first match is found. The following polices must be rewritten before upgrade: ".join(",",@policies_with_final_match)); } # GNA enabled if ($gna_enabled) { # Determine the number of nodes with SSD storage not used as L3 cache my @nodes_without_ssd = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$lnn}{'SSD Storage'}; next unless $NODE_INFO{$lnn}{'SSD Storage'} =~ /No SSDs|No Storage SSDs|Diskless|L3/; push(@nodes_without_ssd, $lnn); } # Calculate the percent of nodes which are being used for SSD storage my $percent_ssd_nodes = 0; if(@nodes_without_ssd) { $percent_ssd_nodes = 100 * ($CLUSTER_INFO{'Node Count'} - scalar(@nodes_without_ssd))/$CLUSTER_INFO{'Node Count'}; debug("checkFilepoolPolicies: Percent SSD Nodes: $percent_ssd_nodes"); } # A minimum of 20% must be SSD storage nodes not used for L3 cache if ($percent_ssd_nodes < 20) { warning("Cluster is enabled for GNA but only ${percent_ssd_nodes}% of all nodes have SSD storage, the minimum requirement is 20%"); } # Calculate the percent of storage which is SSD storage my $percent_ssd_storage = 0; if ($CLUSTER_INFO{'SSD Size'} and $CLUSTER_INFO{'HDD Size'}) { $percent_ssd_storage = sprintf("%.1f", 100 * $CLUSTER_INFO{'SSD Size'}/$CLUSTER_INFO{'HDD Size'}); debug("checkFilepoolPolicies: Percent SSD Storage: $percent_ssd_storage"); } # A minimum of 1.5% of storage must be SSD storage if ($percent_ssd_storage < 1.5) { warning("Cluster is enabled for GNA but only ${percent_ssd_storage}% of all storage is SSD storage, the minimum requirement is 1.5%"); } # Flag an issue for upgrades from prior to 7 to 7 or later where GNA would be deactivated as part of the upgrade if (compareVersions($CLUSTER_INFO{'OneFS'}, "7") < 0 and compareVersions($RPS{'Target'}, "7") >= 0 and ($percent_ssd_nodes < 20 or $percent_ssd_storage < 1.5)) { fail("Cluster does not meet minimum requirements for Global Namespace Acceleration and an upgrade would deactivate Global Namespace Acceleration. Either disable GNA, or call Isilon support. Important: do NOT bypass this check if you are also failing the CheckDiskpoolsMembership check or any other upgrade check.") if $percent_ssd_storage < 1.5; } # Warn if force spill over is enabled which could be bad warning("Disk Pools are configured to spill over and write data to SSD drives."); } addCustomerInformation($_) foreach (@FAIL); printResults(); } # End checkFilepoolPolicies sub checkFileSystemFormatVersion { return if wasCalled(); my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array sysctl efs.gmp.fs_fmt_version"); } else { %gather = gatherCluster("2>&1 grep -H fs_fmt_version */efs.gmp.current_info"); } my @zero_fs_fmt_version = (); my $fs_fmt_version_odd = 0; verboseHeader("checkFileSystemFormatVersion"); verbose(sprintf("%5s %5s", "LNN", "fs_fmt_version")); foreach my $lnn (@LNNS) { next unless $gather{$lnn}; foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /^\s{0,8}fs_fmt_version\s*=\s*(\d+)$/ or $line =~ /fs_fmt_version:\s*(\d+)$/; my $fs_fmt_version = $1; verbose(sprintf("%5s %5s", $lnn, $fs_fmt_version)); push(@zero_fs_fmt_version, $lnn) if ($fs_fmt_version eq "0"); $fs_fmt_version_odd = 1 if ($fs_fmt_version % 2 == 1); } } # Flag if fs_fmt_version is zero on any nodes if (@zero_fs_fmt_version) { fail("The following nodes are reporting a fs_fmt_version of 0: ". compressRange(@zero_fs_fmt_version)); } # Flag if fs_fmt_version is odd on any nodes if ($fs_fmt_version_odd) { if ($RPS{'Target'}) { fail("An odd fs_fmt_version was found, this is indicative that the Upgrade job has not completed and needs to complete prior to an upgrade to OneFS."); } else { warning("An odd fs_fmt_version was found, this is indicative that the Upgrade job has not completed and needs to complete prior to an upgrade to OneFS."); } } } # End checkFileSystemFormatVersion ### # checkKerberos8000: Checks for an issue with the Kerberos # configuration file for OneFS upgrades targetting 8.0.0.0. $CHECKS{'checkKerberos8000'}{'Description'} = "Checks for an issue with the Kerberos configuration file for OneFS upgrades targetting 8.0.0.0."; sub checkKerberos8000 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { debug("checkKerberos8000: Skipping. Current version already greater than 8.0.0.0."); return; } elsif (!$RPS{'Target'} or compareVersions($RPS{'Target'}, "8.0.0.0") != 0) { debug("checkKerberos8000: Skipping. Target version not 8.0.0.0."); return; } printCheck("Kerberos & OneFS 8.0.0.0"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 cat /etc/likewise-krb5-ad.conf"); } else { @gather = gatherLocal("2>&1 cat local/ifsvar_etc.tar/etc/likewise-krb5-ad.conf"); } # Analyze information for issues. my $missing_permitted_enctypes = 1; foreach my $line (@gather) { debug("checkKerberos8000: $line"); if ($line =~ /permitted_enctypes/) { $missing_permitted_enctypes = 0; debug("Found permitted_enctypes, not affected"); } } # If missing the permitted_enctypes line if ($missing_permitted_enctypes) { fail("Permitted Encryption types must be added to /etc/likewise-krb5-ad.conf prior to a OneFS upgrade to 8.0.0.0.", "Escalate to support and reference Bug 168999 or complete additional pre-upgrade procedures to address the issue.") } # Output Check result printResults(); } # End checkKerberos8000 ### # checkLastZoneID8000: Checks for gaps in access zones that may cause # major issues for OneFS upgrades targeting 8.0.0.0. $CHECKS{'checkLastZoneID8000'}{'Description'} = "Checks for gaps in access zones that may cause major issues for OneFS upgrades targeting 8.0.0.0."; sub checkLastZoneID8000 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { debug("checkLastZoneID8000: Skipping. Current version already greater than 8.0.0.0."); return; } elsif (!$RPS{'Target'} or compareVersions($RPS{'Target'}, "8.0.0.0") != 0) { debug("checkLastZoneID8000: Skipping. Target version not 8.0.0.0"); return; } checkAccessZones(); printCheck("LastZoneID & OneFS 8.0.0.0"); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 cat /ifs/.ifsvar/main_config.gc"); } else { @gather = gatherLocal("2>&1 cat local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } # Default lastzoneid to 1 for cases where they have never added/deleted any access zones my $lastzoneid = 1; foreach (@gather) { $lastzoneid = $1 if (/.*lastzoneid.*\"(.*)\"/i); } # If the total number of access zones does not match the lastzoneid fail("SMB share and NFS export configuration may be lost after an upgrade to 8.0.0.0, review KB480190 for details.") if ($CLUSTER_INFO{'Access Zones'} and $CLUSTER_INFO{'Access Zones'} ne $lastzoneid); addCustomerInformation(@FAIL); # Output Check result printResults(); } # End checkLastZoneID8000 ### # checkLegacyLDAP: Checks for Legacy LDAP being enabled in OneFS 6 to OneFS 7 upgrades $CHECKS{'checkLegacyLDAP'}{'Description'} = "Checks for Legacy LDAP being enabled in OneFS 6 to OneFS 7 upgrades"; sub checkLegacyLDAP { return if wasCalled(); return unless ($RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "7") < 0 and compareVersions($RPS{'Target'}, "7") >= 0); printCheck("Legacy LDAP"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 cat /etc/mcp/override/ldap.xml"); } else { @gather = gatherLocal("2>&1 cat local/ifsvar_etc.tar/etc/mcp/override/ldap.xml"); } # Analyze information for issues. foreach my $line (@gather) { fail("Legacy LDAP cannot be upgraded. Please migrate all settings under 'Legacy LDAP' in the WebUI to the 'LDAP' tab and validate correct LDAP functionality before upgrading.") if ($line =~ /modify-attribute.*id="ldap".*attribute="enable".*value="1"/); } addCustomerInformation("Legacy LDAP cannot be upgraded. Please migrate all settings under 'Legacy LDAP' in the WebUI to the 'LDAP' tab and validate correct LDAP functionality before upgrading.") if @FAIL; # Output Check result printResults(); } # End checkLegacyLDAP sub checkNFS8004 { return if wasCalled(); if ($RPS{'Target'}) { return unless $RPS{'Target'} eq "8.0.0.4"; checkUpgradePath(); my $current_code = $CLUSTER_INFO{'OneFS'}; my $affected = 0; foreach my $target_code (sort {compareVersions($a, $b)} @{$RPS{'Upgrade Path'}}) { if (compareVersions($current_code, "7.2") == 0 and compareVersions($target_code, "8.0.0.4") == 0) { $affected = 1; } $current_code = $target_code; } return unless $affected; } else { return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") == 0; } gatherNFSExport(); return unless $CLUSTER_INFO{'NFS Export'}; # Post upgrade issue, NFS Export has - as the Security Type my @blank_export_ids = (); my @unix_export_ids = (); foreach my $id (keys %{$CLUSTER_INFO{'NFS Export'}}) { next unless $CLUSTER_INFO{'NFS Export'}{$id}{'Security Type'}; if ($CLUSTER_INFO{'NFS Export'}{$id}{'Security Type'} =~ /^\s*-\s*$/) { push(@blank_export_ids, $id); debug("checkNFS8004: Export ID $id impacted!"); } if ($CLUSTER_INFO{'NFS Export'}{$id}{'Security Type'} =~ /^\s*unix\s*$/) { push(@unix_export_ids, $id); debug("checkNFS8004: Export ID $id is unix."); } } if (@blank_export_ids and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") == 0) { fail("The following NFS Export IDs are impacted by KB496341: ".join(",",@blank_export_ids)); fail("Review the KB article and complete resolution steps for Post upgrade on 8.0.0.4"); } if (@unix_export_ids and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { warning("The following NFS Export IDs have unix Security Type, may be impacted by KB496341: ".join(",",@unix_export_ids)); warning("Prior to an upgrade to 8.0.0.4, review the KB article and complete pre-upgrade resolution steps"); addRequiredFiles("NFS Export Security Repair Script"); } } # End checkNFS8004 sub checkNonSystemPools { return if wasCalled(); unless ($RPS{'Target'} and compareVersions($RPS{'Target'}, "7.1.1") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") < 0) { debug("checkNonSystemPools: Skipped.."); return; } gatherNetworkPools(); my $non_system_zone_pools = 0; foreach my $id (keys %{$CLUSTER_INFO{'Network Pools'}}) { next unless $CLUSTER_INFO{'Network Pools'}{$id}{'Access Zone'}; if ($CLUSTER_INFO{'Network Pools'}{$id}{'Access Zone'} ne "System") { # Flag issue $non_system_zone_pools = 1; debug("checkNonSystemPools: !!! Network pool $id configured with access zone $CLUSTER_INFO{'Network Pools'}{$id}{'Access Zone'}"); } } if ($non_system_zone_pools) { warning("NFS clients accessing the cluster via any non-System access zone network pools will experience disruption during the upgrade."); warning("NFS exports will default to the System access zone after upgrading to OneFS 7.2 or later."); warning("NFS access is only available via System access zone network pools in OneFS 7.1.1."); warning("CLI and WebUI administration is only available via System access zone network pools starting in OneFS 7.1.1."); warning("HTTP and FTP access is only available via System access zone network pools starting in OneFS 7.1.1."); } } # End checkNonSystemPools $CHECKS{'checkOneFSVersions'}{'Description'} = "Checks running version and target version for any issues. Failure: Any version mismatch across all nodes"; sub checkOneFSVersions { return if wasCalled(); printCheck("OneFS Version"); my %onefs_versions = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); verbose($NODE_INFO{$lnn}{'uname'}) if $NODE_INFO{$lnn}{'uname'}; if (!$NODE_INFO{$lnn}{'OneFS'}) { missing($lnn); next; } # Compile a list of OneFS versions as keys in a hash $onefs_versions{$NODE_INFO{$lnn}{'OneFS'}} = 1; } # Fail if multiple OneFS versions found fail("Multiple OneFS versions detected: ".join(", ", keys %onefs_versions)) if scalar(keys %onefs_versions) > 1; if ($CLUSTER_INFO{'Beta'}) { if ($RPS{'Target'}) { fail("BETA version detected, OneFS upgrades are not supported from BETA versions"); delete $RPS{'Target'}; } else { warning("BETA version detected, OneFS upgrades are not supported from BETA versions"); } } # Info if current version of OneFS is EOSL or approaching EOSL if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") <= 0) { info("The current version of OneFS $CLUSTER_INFO{'OneFS'} is ". "End-of-Service-Life."); info("For details see: https://support.emc.com/docu45445_Isilon-Product-Availability.pdf"); } # elsif (compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") <= 0) { # info("The current version of OneFS $CLUSTER_INFO{'OneFS'} will reach ". # "End-of-Service-Life on 1/31/2019."); # info("For details see: https://support.emc.com/docu45445_Isilon-Product-Availability.pdf"); #} if ($RPS{'F010517EE'}) { if (($CLUSTER_INFO{'OneFS'} =~ /^7/ and compareVersions($CLUSTER_INFO{'OneFS'}, "7.2.1.6") < 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") < 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8\.0\.1\.0/)) { $RPS{'FCO'}{'F010517EEA'}{'Severity'} = "INFO"; @{$RPS{'FCO'}{'F010517EEA'}{'Message'}} = ("Current OneFS $CLUSTER_INFO{'OneFS'} does not meet requirements for FCO F010517EE"); } } # Refer to KB for details info("Refer to KB210503 (https://support.emc.com/kb/210503) for details.") if @WARN or @FAIL; # Output results printResults(); } # End checkOneFSVersion sub checkOverlap { return if wasCalled(); unless ($RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "7") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") < 0 and compareVersions($RPS{'Target'}, "7.1.1") >= 0 and compareVersions($RPS{'Target'}, "8") < 0) { debug("checkOverlap: Skipped.."); return; } # Gather SMB Share information from cluster or from logset debug("checkOverlap: gathering SMB information"); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi_classic smb dump"); } else { @gather = gatherLocal("2>&1 cat local/isi_smb"); } # Populate a hash of share name -> share path my %SMB_Shares = (); my ($share_path, $share_name) = ""; foreach (@gather) { last if /SMB Share Permissions:/; $share_name = $1 if /^Name:\s*(\S.*)/ or /^\s*Share Name:\s*(\S.*)/; $share_path = $1 if /^Path:\s*(\S.*)/ or /^\s*Path:\s*(\S.*)/; next unless $share_name and $share_path; $SMB_Shares{$share_name} = $share_path; ($share_name, $share_path) = ""; } # Gather Access Zone information debug("checkOverlap: gathering Access Zone information"); @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 cat /ifs/.ifsvar/main_config.gc"); } else { @gather = gatherLocal("2>&1 cat local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } # Populate an array of SMB shares associated with each zone my $zone_name = ""; foreach (@gather) { next if !/Zones/; $zone_name = $1 if /\.Zones\._key\.[^\.]*\._name.*\"(.*)\"/; next unless $zone_name; if (/ShareList\.value .*?\"\[ (.*) \]\"/ and !$CLUSTER_INFO{'Zones'}{$zone_name}{'All SMB Shares'}) { my $sharelist = $1; $sharelist =~ s/\\\"//g; my @sharelist = split(", ", $sharelist); foreach (@sharelist) { my $share_name = $1 if /^(.*?):/; next unless $share_name and $SMB_Shares{$share_name}; push(@{$CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}}, $share_name); debug("CLUSTER_INFO->Zones->${zone_name}->Share List=$share_name"); } } if (/AllSMBShares\.value .* \"1\"/) { $CLUSTER_INFO{'Zones'}{$zone_name}{'All SMB Shares'} = 1; debug("CLUSTER_INFO->Zones->${zone_name}->All SMB Shares=1"); @{$CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}} = keys %SMB_Shares; } } debug("checkOverlap: For each access zone, find the lowest common path for the SMB shares associated with it."); my @lowest_common_paths = (); foreach my $zone_name (keys %{$CLUSTER_INFO{'Zones'}}) { next if $zone_name eq "System"; debug("checkOverlap: findLowestCommonPath: $zone_name"); $CLUSTER_INFO{'Zones'}{$zone_name}{'Lowest Common Path'} = findLowestCommonPath(@SMB_Shares{@{$CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}}}) if ($CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}); push(@lowest_common_paths, "$zone_name -> ".$CLUSTER_INFO{'Zones'}{$zone_name}{'Lowest Common Path'}) if ($CLUSTER_INFO{'Zones'}{$zone_name}{'Lowest Common Path'}); } info("Lowest common SMB share paths for each Access Zone are as follows:", @lowest_common_paths) if @lowest_common_paths; debug("checkOverlap: Check each zones shares against every other zones shares for nested shares"); my $nested_share = 0; my $shared_share = 0; foreach my $zone_a (keys %{$CLUSTER_INFO{'Zones'}}) { next if $zone_a eq "System"; foreach my $zone_b (keys %{$CLUSTER_INFO{'Zones'}}) { next if $zone_b eq "System" or $zone_a eq $zone_b; foreach my $share_a (@{$CLUSTER_INFO{'Zones'}{$zone_a}{'Share List'}}) { foreach my $share_b (@{$CLUSTER_INFO{'Zones'}{$zone_b}{'Share List'}}) { if ($SMB_Shares{$share_a} eq $SMB_Shares{$share_b}) { $shared_share = 1; debug("checkOverlap: !!! $share_a associated with $zone_a and $zone_b"); } elsif ($SMB_Shares{$share_a} =~ /^$SMB_Shares{$share_b}/) { $nested_share = 1; debug("checkOverlap: !!! ${share_a}:$SMB_Shares{$share_a} in $zone_a nested within ${share_b}:$SMB_Shares{$share_b} of $zone_b"); } } } } } if ($nested_share or $shared_share) { warning("Detected one or more SMB shares associated with multiple access zones where the share paths are equal or nested. This configuration is no longer supported. Upgrade will succeed, but you will be unable to create new access zones until the path overlap is fixed."); } } # End checkOverlap ### # checkSBR8000: Checks if SBR is enabled pre-upgrade for OneFS upgrades # targeting 8.0.0.0/1. $CHECKS{'checkSBR8000'}{'Description'} = "Checks if SBR is enabled pre-upgrade for OneFS upgrades targeting 8.0.0.0/1."; sub checkSBR8000 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.2") >= 0) { debug("checkSBR8000: Skipping. Current version greater than or equal to 8.0.0.2."); return; } elsif (compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") < 0) { debug("checkSBR8000: Skipping. Current version less than 7.2."); return; } elsif (!$RPS{'Target'} or $RPS{'Target'} !~ /^8.0.0.[01]$/) { debug("checkSBR8000: Skipping. Target version not 8.0.0.0 or 8.0.0.1."); return; } printCheck("SBR & OneFS 8.0.0.0/1"); # Gather information from cluster or from logset my @gather = (); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi network external view"); } else { @gather = gatherLocal("2>&1 cat local/isi_networks_external"); } } else { if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi networks"); } else { @gather = gatherLocal("2>&1 cat local/isi_networks"); } } # Analyze information for issues. my $source_based_routing_enabled = 0; foreach my $line (@gather) { if ($line =~ /Source Based.*?enabled$/i or $line =~ /Source Based.*?true$/i) { debug("checkSBR8000: Source Based Routing is enabled"); $source_based_routing_enabled = 1; } } # If Source Based Routing is in use. if ($source_based_routing_enabled) { fail("Source Based Routing is enabled. An issue exists in 8.0.0.0 and 8.0.0.1 where when source based routing is enabled, nodes will not reply outside of their subnet. This could potentially leave the cluster in an unreachable state. It is not recommended to upgrade to either of these versions with SBR enabled. This issue is due to be fixed in the release of 8.0.0.2. Alternatively, if upgraded to 8.0.0.0 or 8.0.0.1, a fix is available in patch-169853."); } # Output Check result printResults(); } # End checkSBR8000 ### # checkUpgradePath: Checks where multiple jumps are required and provides specifics required. $CHECKS{'checkUpgradePath'}{'Description'} = "Checks where multiple jumps are required and provides specifics required."; sub checkUpgradePath { return if wasCalled(); # If there is no target version, skip this! return if !$RPS{'Target'}; checkPatches(); printCheck("Upgrade Path"); # Fail if target code is EOSL if ($RPS{'Target'} and compareVersions($RPS{'Target'}, "7.2") < 0) { warning("The target OneFS version is End-of-Service-Life."); addCustomerInformation("The target OneFS version, $RPS{'Target'}, is End-of-Service-Life."); } # Flag if currently running a beta version if ($CLUSTER_INFO{'Beta'}) { delete $RPS{'Target'}; printResults(); return; } # Generate an upgrade path @{$RPS{'Upgrade Path'}} = generateUpgradePaths($CLUSTER_INFO{'OneFS'}, $RPS{'Target'}); debug("checkUpgradePath: ".join(", ", @{$RPS{'Upgrade Path'}})); # Fail if we couldn't find any path if (scalar(@{$RPS{'Upgrade Path'}}) <= 1) { fail("No found upgrade path from $CLUSTER_INFO{'OneFS'} to $RPS{'Target'}"); delete $RPS{'UpgradePlan'}; } # Warn if target code requires multiple jumps if (scalar(@{$RPS{'Upgrade Path'}}) >= 3) { warning("The requested target version will require multiple upgrade jumps with the following recommended steps: ".join(" -> ", @{$RPS{'Upgrade Path'}})); addCustomerInformation("Upgrade Plan", "The requested target version will require multiple upgrade jumps with the following recommended steps: ".join(" -> ", @{$RPS{'Upgrade Path'}})); } my ($source, $target) = ""; foreach (@{$RPS{'Upgrade Path'}}) { $target = $_; if ($target and $source) { $RPS{'OneFS Upgrade'}{$source}{'Target'} = $target; if (isUpgradeSimultaneous($source, $target)) { warning("The upgrade from $source to $target must be performed as a simultaneous upgrade"); addCustomerInformation("Upgrade Plan", "The upgrade from $source to $target must be performed as a simultaneous upgrade"); } if (isUpgradeRestripe($source, $target)) { warning("After the upgrade from $source to ${target}, an 'Upgrade' data restriping job will run."); addCustomerInformation("After the upgrade from $source to ${target}, an 'Upgrade' data restriping job will run."); } } $source = $target; } # Warn if upgrading to 8.0.1.0 in regards to being simultaneous. if ($RPS{'Target'} and $RPS{'Target'} =~ /^8\.0\.1\.0/) { warning("Upgrades to 8.0.1.0 should only be performed simultaneous to avoid disruptions caused by an issue identified in 8.0.1.0 code level that is addressed in a disruptive patch."); } # Inform customers/warn about OneFS 8.0.1 being withdrawn and recommend other branches. if ($RPS{'Target'} and $RPS{'Target'} =~ /^8\.0\.1\./) { addCustomerInformation(warning("Consider instead performing a OneFS upgrade to the target in the OneFS 8.0.0 branch or to a OneFS 8.1 version.")); } # For all upgrades from OneFS 8.0 to OneFS 8.1, disable isi_drive_repurpose_d service during the upgrade. if ($RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") == 0 and $RPS{'Target'} =~ /^(8\.1\.0\.[012]|8\.1\.1\.0)$/) { critical("During an upgrade from OneFS $CLUSTER_INFO{'OneFS'} to $RPS{'Target'} the isi_drive_repurpose_d service should be disabled to prevent an issue where drives incorrectly smartfail."); } # For all upgrades from prior to OneFS 8.1 to OneFS 8.1 and later if ($RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") < 0 and compareVersions($RPS{'Target'}, "8.1") >= 0) { addCustomerInformation(warning("After the upgrade from OneFS $CLUSTER_INFO{'OneFS'} to OneFS $RPS{'Target'}, a one-time license conversion to a software license must be performed, and, following the license update, if using an SRS gateway for remote connectivity the SRS configuration on the cluster should be updated. A quick reference on how to do this is available at KB 525065 (https://support.emc.com/kb/525065)")); } printResults(); } # End checkUpgradePath ### # findLowestCommonPath: Finds the 'lowest' common path given an array of paths. # Used to give a guess at what the base directory should be for each access zone. sub findLowestCommonPath { debug("findLowestCommonPath: Arguments [ ".join(", ", @_)." ]"); my @lowest_common_path_array = (); foreach my $share_path (@_) { my @share_path_array = split("/", $share_path); if (!@lowest_common_path_array) { @lowest_common_path_array = @share_path_array; next; } for (my $i = 0; $i <= $#lowest_common_path_array; $i++) { if ($i > $#share_path_array or $share_path_array[$i] ne $lowest_common_path_array[$i]) { splice(@lowest_common_path_array, $i); last; } } } my $lowest_common_path = join("/", @lowest_common_path_array); debug("findLowestCommonPath: Results [ $lowest_common_path ]"); return $lowest_common_path; } # End findLowestCommonPath ### # gatherForceSpill: Checks if force spill is enabled sub gatherForceSpill { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} grep -h spill /etc/mcp/override/sysctl.conf"); } else { @gather = gatherLocal("2>&1 grep -h spill local/ifsvar_etc.tar/etc/mcp/override/sysctl.conf"); } # Analyze information for issues. my $force_spill; foreach my $line (@gather) { if ($line =~ /disk_pool_global_force_spill.*1$/ or $line =~ /disk_pool_global_spill.*1$/) { $CLUSTER_INFO{'Force Spill'} = 1; debug("CLUSTER_INFO->Force Spill=1"); last; } } } # End gatherForceSpill ### # gatherOneFSVersions: Gathers OneFSVersion from all nodes. sub gatherOneFSVersions { return if wasCalled(); # Gather information my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array uname -a"); } else { %gather = gatherCluster("2>&1 grep -H . */uname"); } # Assign variables foreach my $lnn (sort {$a <=> $b} keys %gather) { foreach my $line (@{$gather{$lnn}}) { $NODE_INFO{$lnn}{'uname'} = $line; debug("NODE_INFO->${lnn}->uname=$line"); if ($line =~ /BETA/) { $line =~ s/BETA\.//g; $CLUSTER_INFO{'Beta'} = 1; debug("CLUSTER_INFO->Beta=1") } if ($line =~ /^\s*Isilon OneFS \S+ v(\S*) /) { $NODE_INFO{$lnn}{'OneFS'} = $1; debug("NODE_INFO->${lnn}->OneFS=$1"); if (!$CLUSTER_INFO{'OneFS'} or compareVersions($CLUSTER_INFO{'OneFS'}, $1) < 0) { $CLUSTER_INFO{'OneFS'} = $1; debug("CLUSTER_INFO->OneFS=$CLUSTER_INFO{'OneFS'}"); } } } } # Abort if unable to determine what OneFS version is currently in use if (!$CLUSTER_INFO{'OneFS'}) { fail("Unable to determine a OneFS version, checks will not proceed."); printCheck("Gathering OneFS Versions"); printResults(); exit; } } # End gatherOneFSVersions ### # generateTargetVersions: Generates an array of OneFS versions within # a specific family that are greater than or equal to the provided # version. sub generateTargetVersions { return unless $_[0]; our %Code_Matrix; my $version = $_[0]; debug("generateTargetVersions: $version"); my $version_family = $1 if ($version =~ /^(.*)\.\d+$/); my @version_array = (); foreach (keys %Code_Matrix) { push(@version_array, $_) if (compareVersions($_, $version_family) == 0 and compareVersions($_, $version) >= 0); } return @version_array; } # End generateTargetVersions ### # generateUpgradePaths: Generates an array for the upgrade path sub generateUpgradePaths { unless ($_[0] and $_[1]) { debug("generateUpgradePaths: Not enough arguments provided"); return 0; } our %Code_Matrix; # source will originally be the current OneFS version and then will iterate # through the list of valid targets from the current OneFS version if # multiple jumps are needed until a valid path is found my $source = $_[0]; # target will always be the target for the OneFS upgrade that the path is being generated for. my $target = $_[1]; debug("generateUpgradePaths: $source $target"); # The first call to generateUpgradePaths if ($target eq $RPS{'Target'} and $source eq $CLUSTER_INFO{'OneFS'}) { # Use 7.1.1.11 as an intermediary step when going to 8.1.1 or later if coming from a version prior to 7.1.1.11. if (compareVersions($RPS{'Target'}, "8.1.1") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1.11") < 0) { my @target_path_to_711 = generateUpgradePaths($CLUSTER_INFO{'OneFS'}, "7.1.1.11"); my @target_path_from_711 = generateUpgradePaths("7.1.1.11", $RPS{'Target'}); # If for some reason no upgrade path is found, return 0 return 0 unless $target_path_to_711[0] and $target_path_from_711[0]; # Combine the two paths and return it pop(@target_path_to_711); push(@target_path_to_711, @target_path_from_711); return @target_path_to_711; } } # Multiple jumps needed if ($target and $source and $RPS{'Target'} and $CLUSTER_INFO{'OneFS'} and $source ne $CLUSTER_INFO{'OneFS'}) { # Avoid OneFS 7.1.1.7/8 as an intermediary step when going to 7.2.1.3. if ($target =~ /^7\.2\.1\.3$/ and $source =~ /^7\.1\.1\.[89]$/) { return 0; } # Avoid OneFS 8.0.1.0 as an intermediary step, we don't like 8.0.1.0. if ($source =~ /^8\.0\.1\.0$/) { return 0; } # Avoid OneFS 8.0 as an intermediary step when going from less than 7.1.1 to 8.1.0 or higher if (compareVersions($target, "8.1") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") <= 0 and compareVersions($source, "8.0") == 0) { return 0; } # Avoid OneFS 7.2.0 as an intermediary step when going from less than 7.1.1 to 7.2.1 or higher if (compareVersions($target, "7.2.1") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1") <= 0 and compareVersions($source, "7.2.0") == 0) { return 0; } } # Gather a list of codes that can be upgraded to from $source my @target_array = (); # Check in-family target codes, if target is found return it foreach (generateTargetVersions($source)) { # Skip if target version is less than or equal to source or greater than target next if compareVersions($_, $source) <= 0 or compareVersions($_, $target) > 0; # Return if the target version is found if (compareVersions($_, $target) == 0) { debug("generateUpgradePaths: Found upgrade match: $source -> $target"); return ($source, $target); } # Otherwise add to the target version array push(@target_array, $_); } # Locate the key in the code matrix for out of family upgrades my $source_key = ""; foreach (keys %Code_Matrix) { $source_key = $_ if (compareVersions($source, $_) == 0); } # Generate out-of-family target codes foreach my $minimum_target_version (@{$Code_Matrix{$source_key}}) { # Skip if target version is less than or equal to source or greater than target next if compareVersions($minimum_target_version, $source) <= 0 or compareVersions($minimum_target_version, $target) > 0; foreach (generateTargetVersions($minimum_target_version)) { push(@target_array, $_) if compareVersions($_, $target) <= 0; } } # Sort list of out-of-family target codes @target_array = sort {compareVersions($b, $a)} @target_array; debug("generateUpgradePaths: Target Version Array: ".join(", ", @target_array)); # Search for the target version and return if found foreach (@target_array) { if (compareVersions($_, $target) == 0) { debug("generateUpgradePaths: Found upgrade match: $source -> $target"); return ($source, $target); } } # If the target was not found, it may be a multi-jump upgrade debug("generateUpgradePaths: No match found, searching for multiple jumps."); foreach (@target_array) { my @target_path = (); push(@target_path, generateUpgradePaths($_, $target)); return ($source, @target_path) if ($target_path[0]); } # No paths found, return 0 return 0; } # End generateUpgradePaths sub checkKB520799 { return if wasCalled(); return unless $RPS{'Live'}; return unless hasPatch('patch-208294'); # Gather information from cluster or from logset my %gather = gatherCluster(qq[ 2>&1 ${SUDO} isi_for_array ]. qq[ 'ls /lib/libcrypto.so*' ]); # Analyze information for issues. my @nodes_missing_patched = (); foreach my $lnn (@LNNS) { next unless $gather{$lnn}; my $has_patched = 0; foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /\/lib\/libcrypto\.so\.8\.patched/; $has_patched = 1; } next if $has_patched; push(@nodes_missing_patched, $lnn); } if (@nodes_missing_patched) { fail("patch-208294 is installed but /lib/libcrypto.so.8.patched is ". "missing on the following nodes: ". compressRange(@nodes_missing_patched). ". Follow resolution steps in KB520799 ". "(https://support.emc.com/kb/520799)."); } } # End checkKB520799 # Uses local variables to validate patches sub checkPatch { return unless $patch and $re_versions; return unless (!$RPS{'Target'} and $CLUSTER_INFO{'OneFS'} =~ $re_versions and !hasPatch($patch)) or ($RPS{'Target'} and $RPS{'Target'} =~ $re_versions); $RPS{'Post Upgrade Patches'}{$patch} = (); $RPS{'Post Upgrade Patches'}{$patch}{'Impacts'} = $impacts if $impacts; $RPS{'Post Upgrade Patches'}{$patch}{'Important'} = $important if $important; $RPS{'Post Upgrade Patches'}{$patch}{'Service Restart'} = $service_restart if $service_restart; $RPS{'Post Upgrade Patches'}{$patch}{'Priority'} = $priority if $priority; @{$RPS{'Post Upgrade Patches'}{$patch}{'Reboot'}} = @reboot if @reboot; if ($RPS{'Target'}) { warning("$patch should be applied after OneFS upgrade."); } else { warning("$patch should be applied."); if (@conflicting_patches) { warning("$patch is recommended but is known to conflict with patches: ".join(", ",@conflicting_patches)); warning("If the customer would like to have $patch performed they should review the conflicting patches and determine whether they want to keep them or perform $patch"); foreach my $conflicting_patch (@conflicting_patches) { $RPS{'Deprecated Patches'}{"uninstall $conflicting_patch"}{'Service Restart'} = $service_restart if $service_restart; $RPS{'Deprecated Patches'}{"uninstall $conflicting_patch"}{'Impacts'} = "Please review the README for $conflicting_patch for details on impacts during removal"; @{$RPS{'Deprecated Patches'}{"uninstall $conflicting_patch"}{'Reboot'}} = @reboot if @reboot; } } if (@deprecated_patches) { warning("Prior to installing $patch, the following deprecated patches must be uninstalled: ".join(", ",@deprecated_patches)); foreach my $deprecated_patch (@deprecated_patches) { $RPS{'Deprecated Patches'}{"uninstall $deprecated_patch"}{'Service Restart'} = $service_restart if $service_restart; $RPS{'Deprecated Patches'}{"uninstall $deprecated_patch"}{'Impacts'} = "Please review the README for $deprecated_patch for details on impacts during removal"; @{$RPS{'Deprecated Patches'}{"uninstall $deprecated_patch"}{'Reboot'}} = @reboot if @reboot; } } } } sub checkPatch146974 { local $patch = "patch-146974"; local $re_versions = qr/^(7\.1\.1\.2)$/; local $impacts = "During installation or removal of $patch, the SMB and authentication services are unavailable and SMB client connections to the cluster are disconnected. After the patch is installed, SMB clients must reestablish a connection to the cluster."; checkPatch(); } sub checkPatch147686 { local $patch = "patch-147686"; local $re_versions = qr/^(7\.1\.0\.(4|5|6))$/; local $impacts = "During installation or removal of $patch, the SMB and authentication services are unavailable and SMB client connections to the cluster are disconnected. After the patch is installed, SMB clients must reestablish a connection to the cluster."; checkPatch(); } sub checkPatch161166 { local $patch = "patch-161166"; local $re_versions = qr/^(7\.2\.0\.2)$/; local $impacts = "During installation or removal of $patch, the SMB and authentication services are unavailable and SMB client connections to the cluster are disconnected. After the patch is installed, SMB clients must reestablish a connection to the cluster."; checkPatch(); } sub checkPatch164666 { local $patch = "patch-164666"; local $re_versions = qr/^(7\.1\.1\.(0|1|2|3|4|5|6|7|8)|7\.2\.1\.(0|1|2))$/; checkPatch(); } sub checkPatch175451 { gatherNetworkInterfaces(); local $patch = "patch-175451"; local $re_versions = qr/^(7\.2\.0\.(0|1|2|3|4|5))$/; local $impacts = "During installation or removal of patch 175451, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch178793 { local $patch = "patch-178793"; local $re_versions = qr/^(8\.0\.0\.(0|1))$/; local $impacts = "During installation or removal of $patch, the isi_papi_d process and SmartConnect service are stopped. As a result, CLI commands beginning with the 'isi' prefix cannot be run and both the OneFS web administration interface and the SmartConnect service are unavailable. After installation or removal of this patch, the isi_papi_d process and SmartConnect service restart automatically."; checkPatch(); } sub checkPatch179385 { local $patch = "patch-179385"; local $re_versions = qr/^(8\.0\.0\.2)$/; local $impacts = "During installation or removal of $patch, the isi_papi_d process and SmartConnect service are stopped. As a result, CLI commands beginning with the 'isi' prefix cannot be run and both the OneFS web administration interface and the SmartConnect service are unavailable. After installation or removal of this patch, the isi_papi_d process and SmartConnect service restart automatically."; checkPatch(); } sub checkPatch181490 { local $patch = "patch-181490"; local $re_versions = qr/^(8\.0\.1\.0)$/; local $impacts = "During installation or removal of $patch, the isi_papi_d process and SmartConnect service are stopped. As a result, CLI commands beginning with the 'isi' prefix cannot be run and both the OneFS web administration interface and the SmartConnect service are unavailable. After installation or removal of $patch, the isi_papi_d process and SmartConnect service restart automatically."; checkPatch(); } sub checkPatch188239 { gatherNetworkInterfaces(); local $patch = "patch-188239"; local $re_versions = qr/^(8\.0\.0\.[01234])$/; local $impacts = "During installation or removal of patch 188239, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch188660 { local $patch = "patch-188660"; local $re_versions = qr/^(8\.0\.1\.0)$/; local @deprecated_patches = hasPatch("patch-(187743|186696|185821|184615)"); local $impacts = "During installation or removal of $patch, the service that manages SMB, NFS, HDFS, and Swift client connections as well as authentication to the cluster is restarted. As a result, during installation or removal of $patch, client connections to the cluster over these protocols might be disrupted and new client connections cannot be authenticated. FTP and HTTP connections might be affected in some environments, and client connections might have to be reestablished manually after the installation or removal process is complete."; checkPatch(); } sub checkPatch189132 { gatherNetworkInterfaces(); local $patch = "patch-189132"; local $re_versions = qr/^(7\.1\.1\.(0|1|2|3|4|5|6|7|8|9|10|11))$/; local $impacts = "During installation or removal of patch 189132, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces might be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch189133 { gatherNetworkInterfaces(); local $patch = "patch-189133"; local $re_versions = qr/^(7\.2\.1\.[012345])$/; local @deprecated_patches = hasPatch("patch-(175451)"); local $impacts = "During installation or removal of patch 189133, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch206322 { local $patch = "patch-206322"; local $re_versions = qr/^(8\.0\.0\.4)$/; local @deprecated_patches = hasPatch("patch-(187902|188699|189708|189853|191603|194238|194268|195171|196308|196448|196845|196906|197503|197648|198834|199265)"); local @conflicting_patches = hasPatch("patch-(191606|194795|196968|197390|197710|197902|199123|200631|200950|200951|203704|203895)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: SMB, Likewise, NFS, HDFS, Isi Migrate, CloudPools, Flexnet, SmartConnect, NDMP, PAPI, Tardis"; # Patch 206322 is deprecated by patch 212411 return if !$RPS{'Target'} and $CLUSTER_INFO{'OneFS'} =~ $re_versions and hasPatch("patch-212411"); checkPatch(); } sub checkPatch213286 { local $patch = "patch-213286"; local $re_versions = qr/^(7\.2\.1\.5)$/; local @deprecated_patches = hasPatch("patch-(211400)"); local @conflicting_patches = hasPatch("patch-(200764|205288)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: webui, apache2, sshd"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; checkPatch(); } sub checkPatch213287 { local $patch = "patch-213287"; local $re_versions = qr/^(7\.2\.1\.6)$/; local @conflicting_patches = hasPatch("patch-(200764|205288)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: webui, apache2, sshd"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; checkPatch(); } sub checkPatch219279 { local $patch = "patch-219279"; local $re_versions = qr/^(8\.0\.1\.2)$/; local @deprecated_patches = hasPatch("patch-(216929|216763|215881)"); local @conflicting_patches = hasPatch("patch-(215020|213630)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: SMB, Likewise, NFS, HDFS, SNMP, isi_avscan_d, Migrate, pcnfsd, NDMP, PAPI, Tardis"; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch220098 { local $patch = "patch-220098"; local $re_versions = qr/^(8\.1\.0\.2)$/; local @deprecated_patches = hasPatch("patch-(219689|224219|227398)"); local @conflicting_patches = hasPatch("patch-(222005)"); local $impacts = "During installation or removal of $patch, the following services are restarted: auth, celog, hdfs, isi_hwmon, isi_smartconnect, isi_stats_d, isi_stats_hist_d, lwdm, lwswift, nfs, smb"; local $service_restart = 1; checkPatch(); } sub checkPatch220176 { local $patch = "patch-220176"; local $re_versions = qr/^(8\.0\.1\.2)$/; local @deprecated_patches = hasPatch("patch-(213280)"); local $impacts = "During installation or removal of $patch, the following services are restarted: isi_mcp, apache2, isi_webui"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; checkPatch(); } sub checkPatch221602 { local $patch = "patch-221602"; local $re_versions = qr/^(8\.0\.0\.4)$/; local @conflicting_patches = hasPatch("patch-(208294|205289)"); local $important = "IMPORTANT: If you update Isilon OneFS with patch 221602, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing patch 221602 due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch221603 { local $patch = "patch-221603"; local $re_versions = qr/^(8\.0\.0\.5)$/; local @deprecated_patches = hasPatch("patch-(217637|211167|209258|208294|207787|205289)"); local $impacts = "During installation or removal of $patch, the following services are restarted: mcp, webui, http, isi_papi_d, isi_object_d"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\n IMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch221947 { local $patch = "patch-221947"; local $re_versions = qr/^(8\.1\.0\.1)$/; local @deprecated_patches = hasPatch("patch-(213282|217638)"); local @conflicting_patches = hasPatch("patch-(211962|220053|194276)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch221949 { local $patch = "patch-221949"; local $re_versions = qr/^(8\.1\.0\.0)$/; local @deprecated_patches = hasPatch("patch-(213281)"); local @conflicting_patches = hasPatch("patch-(211967|211961|205291|200762)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch222487 { local $patch = "patch-222487"; local $re_versions = qr/^(8\.0\.1\.1)$/; local @deprecated_patches = hasPatch("patch-(213279|211960)"); local @conflicting_patches = hasPatch("patch-(215497|211403|210691|207276|205290)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch223311 { local $patch = "patch-223311"; local $re_versions = qr/^(8\.0\.0\.5)$/; local @deprecated_patches = hasPatch("patch-(211169|217838)"); local @conflicting_patches = hasPatch("patch-(203466|204429|204812|204900|206113|210031|210400|211170|211795|213989|215592|215685|215703|216731)"); local $impacts = "During installation or removal of $patch, the following services are restarted: NFS, PC NFS Server, SMB, LIKEWISE, HDFS, CloudPools, Tardis, PAPI, SmartConnect, NDMP, Authentication, LWSM, Swift, SNMP, FlexNet, JobEngine, Statistics & Monitoring, Statistics History"; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch223338 { local $patch = "patch-223338"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(208154|214178|215884|219375|219904|221633|224127|224804|227485|231811)"); local @conflicting_patches = hasPatch("patch-(222935|218751)"); local $impacts = "During installation or removal of $patch, the following services are restarted: auth, celog, hdfs, isi_avscan_d, isi_cpool_d, isi_cpool_io_d, isi_flexnet_d, isi_hangdump, isi_job_d, isi_km_d, isi_mcp, isi_papi_d, isi_smartconnect, isi_stats_d, isi_stats_hist_d, lwsm, lwswift, nfs, smb, snmp"; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch224116 { local $patch = "patch-224116"; local $re_versions = qr/^(8\.1\.1\.1)$/; local @deprecated_patches = hasPatch("patch-(222493)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch226323 { local $patch = "patch-226323"; local $re_versions = qr/^(8\.1\.0\.2)$/; local @deprecated_patches = hasPatch("patch-(220928|222464|223838)"); local @conflicting_patches = hasPatch("patch-(223989)"); local $service_restart = 1; local $priority = 10; local @reboot = @LNNS; checkPatch(); } sub checkPatch226984 { local $patch = "patch-226984"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(225079)"); local $service_restart = 1; local $priority = 10; local @reboot = @LNNS; checkPatch(); } sub checkPatch228458 { local $patch = "patch-228458"; local $re_versions = qr/^(8\.1\.0\.2)$/; local @deprecated_patches = hasPatch("patch-(213283|219709|221946|223678)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch229156 { local $patch = "patch-229156"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(215690|221638)"); local $impacts = "During installation or removal of $patch, the following services are restarted: SyncIQ, NDMP, PAPI, MCP"; local $service_restart = 1; checkPatch(); } sub checkPatch229158 { local $patch = "patch-229158"; local $re_versions = qr/^(8\.0\.0\.5)$/; local @deprecated_patches = hasPatch("patch-(203734|211168|203743|213179|217463|224399)"); local @conflicting_patches = hasPatch("patch-(206113|210400|211169|217838)"); local $impacts = "During installation or removal of $patch, the following services are restarted: SyncIQ, PAPI"; local $service_restart = 1; checkPatch(); } sub checkPatch229160 { local $patch = "patch-229160"; local $re_versions = qr/^(8\.1\.0\.2)$/; local $impacts = "During installation or removal of $patch, the following services are restarted: SyncIQ, PAPI, NDMP"; local $service_restart = 1; local @deprecated_patches = hasPatch("patch-(222718)"); checkPatch(); } sub checkPatch231386 { local $patch = "patch-231386"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(213278|221617|223243)"); local @conflicting_patches = hasPatch("patch-(228563)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://support.emc.com/kb/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch233663 { local $patch = "patch-233663"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @deprecated_patches = hasPatch("patch-(232890|234237)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: celog, isi_cpool_d, isi_cpool_io_d, isi_drive_repurpose_d, isi_job_d, isi_migrate, isi_ph_rpcd, isi_smartpools_d, ndmpd, isi_papi_d, auth, hdfs, isi_mcp, isi_tardis_gcfg_d, isi_tardis_d, lwsm, lwswift, nfs, smb"; local $important = "The OneFS Protocols/Userspace Rollup patch, $patch, should be considered mandatory for every upgrade to OneFS 8.0.0.7."; local $service_restart = 1; local $priority = 10; checkPatch(); } sub checkPatch8007Kernel { local $patch = "patch-240795"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @conflicting_patches = hasPatch("patch-(233621)"); local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch8007Userspace { local $patch = "patch-233663"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @deprecated_patches = hasPatch("patch-(232890|234237)"); local @conflicting_patches = hasPatch("patch-(241945|240514)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: celog, isi_cpool_d, isi_cpool_io_d, isi_drive_repurpose_d, isi_job_d, isi_migrate, isi_ph_rpcd, isi_smartpools_d, ndmpd, isi_papi_d, auth, hdfs, isi_mcp, isi_tardis_gcfg_d, isi_tardis_d, lwsm, lwswift, nfs, smb"; local $important = "The OneFS Protocols/Userspace Rollup patch, $patch, should be considered mandatory for every upgrade to OneFS 8.0.0.7."; local $service_restart = 1; local $priority = 10; checkPatch(); } sub checkPatch8104Kernel { local $patch = "patch-242570"; local $re_versions = qr/^(8\.1\.0\.4)$/; local @deprecated_patches = hasPatch("patch-(228202|235853|236488|236994|240815)"); local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch8104Userspace { return if hasPatch("patch-(242136|241584)"); # GA RUP details local $patch = "patch-241584"; local $re_versions = qr/^(8\.1\.0\.4)$/; local @conflicting_patches = hasPatch("patch-(222280|228133|233356|234345|234538|235934|239173|241946|242136)"); local @deprecated_patches = hasPatch("patch-(239173|239649)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: auth, celog, hdfs, isi_avscan_d, isi_mcp, lwsm, lwswift, nfs, smb, isi_quota_notify_d"; local $service_restart = 1; local $priority = 10; # DA RUP details if (hasPatch("patch-(228133|234538)")) { $patch = "patch-242136"; @conflicting_patches = hasPatch("patch-(222280|231017|231712|231786|232711|233356|234345|235853|235862|235934|236576|237861|239173|239649|241584|241946)"); @deprecated_patches = hasPatch("patch-(228133|234538)"); $impacts = "During the installation of removal of $patch, the following services are restarted: apache2, auth, celog, hdfs, isi_avscan_d, isi_cpool_d, isi_cpool_io_d, isi_drive_repurpose_d, isi_hwmon, isi_job_d, isi_mcp, isi_migrate, isi_papi_d, isi_ph_rpcd, isi_smartpools_d, isi_smartquotas, isi_stats_d, isi_tardis_d, isi_tardis_gcfg_d, isi_webui, lwsm, lwswift, ndmpd, nfs, rpcbind, smb, snmp, isi_flexnet_d"; } checkPatch(); } sub checkPatch8120Kernel { local $patch = "patch-243897"; local $re_versions = qr/^(8\.1\.2\.0)$/; local @deprecated_patches = hasPatch("patch-(237133|237483|240794)"); local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch8120Userspace { return if hasPatch("patch-(243894|243896)"); # GA RUP details local $patch = "patch-243894"; local $re_versions = qr/^(8\.1\.2\.0)$/; local @deprecated_patches = hasPatch("patch-(236288|239343|240163|241940|241947|242854)"); local @conflicting_patches = hasPatch("patch-(239260|240798|243896)"); local $impacts = "During installation or removal of this patch, the following services will be stopped and restarted automatically: isi_group_change_d, celog, isi_tardis_gcfg_d, isi_cpool_io_d, isi_cpool_d, isi_object_d, hdfs, isi_hangdump, lwswift, isi_webui, isi_papi_d, ndmpd, apache2, isi_km_d, isi_migrate, isi_drivemon, isi_smartpools_d, isi_tardis_d, isi_drive_repurpose_d, isi_audit_cee, isi_audit_syslog, isi_job_d, isi_audit_d, isi_upgrade_d, nfs"; local $service_restart = 1; local $priority = 10; # DA RUP details if (hasPatch("patch-(240798)")) { $patch="patch-243896"; @deprecated_patches = hasPatch("patch-(236288|239260|239343|240163|240798|241940|241947|242854|243894)"); @conflicting_patches = hasPatch("patch-(237113)"); $impacts = "During installation of removal of this patch, the following services will be stopped and restarted automatically: apache2, auth, celog, hdfs, isi_audit_cee, isi_audit_d, isi_audit_syslog, isi_cpool_d, isi_cpool_io_d, isi_drive_repurpose_d, isi_drivemon, isi_esrs_d, isi_group_change_d, isi_hangdump, isi_hwmon, isi_job_d, isi_km_d, isi_mcp, isi_migrate, isi_object_d, isi_papi_d, isi_ph_rpcd, isi_rsapi_d, isi_smartpools_d, isi_tardis_d, isi_tardis_gcfg_d, isi_upgrade_d, isi_webui, lwsm, lwswift, ndmpd, nfs, smb"; } checkPatch(); } $CHECKS{'checkPatches'}{'Description'} = "Checks for highly recommended patches, on the current version if no target version, or for the target version."; sub checkPatches { return if wasCalled(); checkKB521778(); printCheck("Highly Recommended Patches"); gatherPatches(); gatherNetworkInterfaces(); # Display all installed patches my @patches = hasPatch(); if (@patches) { details("Installed Patches: ".join(", ", @patches)); } # Check for and display partially installed patches my $all_nodes = compressRange(@LNNS); my @partially_installed = (); foreach my $patch (@patches) { next if $CLUSTER_INFO{'Patches'}{$patch} eq $all_nodes; push(@partially_installed, "$patch ($CLUSTER_INFO{'Patches'}{$patch})"); } if (@partially_installed) { warning("The following patches are not installed across all nodes in the cluster: ".join(", ", @partially_installed)); } # Check if any IsiFru, IsiHw, IsiFw, or Drive_Support packages show installed on any nodes my @temporary_patches = hasPatch("IsiFru_Package|IsiHw_Package|Drive_Support|Drive_Firmware|IsiFw"); if (@temporary_patches) { critical("The following patches were found as installed and should be uninstalled: ".join(", ",@temporary_patches)); } # Check if any of the EncodingCheck job patches are installed my ($encoding_patch) = hasPatch("patch-(185213|185212|185211|185210|185209|185207|185204|185203|176073|176331|176068|176332|176069|176334|176335|176336|176236|176070|176461|176460|176459|176458)"); if ($RPS{'Target'} and $encoding_patch) { critical("Prior to performing any OneFS upgrade, patch $encoding_patch, must be uninstalled from the cluster."); } # Check if Patch 208294 is installed but missing /lib/libcrypto.so.8.patched checkKB520799(); # Check if any patches are listed as installing checkPatchesInstalling(); # Do not make patch recommendations when called via HealthCheck if ($RPS{'JSON'}) { printResults(); return; } # Check for patches checkPatchHC812(); checkPatch8120Userspace(); checkPatch8120Kernel(); checkPatchHC810(); checkPatch8104Userspace(); checkPatch8104Kernel(); checkPatchHC8007(); checkPatch8007Userspace(); checkPatch8007Kernel(); checkPatch231386(); checkPatch229160(); checkPatch229158(); checkPatch229156(); checkPatch228458(); checkPatch226984(); checkPatch226323(); checkPatch224116(); checkPatch223338(); checkPatch223311(); checkPatch222487(); checkPatch221949(); checkPatch221947(); checkPatch221603(); checkPatch221602(); checkPatch220176(); checkPatch220098(); checkPatch219279(); checkPatch213287(); checkPatch213286(); checkPatch206322(); checkPatch189133(); checkPatch189132(); checkPatch188660(); checkPatch188239(); checkPatch181490(); checkPatch179385(); checkPatch178793(); checkPatch175451(); checkPatch164666(); checkPatch161166(); checkPatch147686(); checkPatch146974(); # If there is no OneFS upgrade being performed, the patches should be Pre Upgrade Patches, not Post Upgrade Patches. if (!$RPS{'Target'} and $RPS{'Post Upgrade Patches'}) { if (!$RPS{'Pre Upgrade Patches'}) { $RPS{'Pre Upgrade Patches'} = $RPS{'Post Upgrade Patches'}; } else { foreach my $patch (keys %{$RPS{'Post Upgrade Patches'}}) { next if $RPS{'Pre Upgrade Patches'}{$patch}; $RPS{'Pre Upgrade Patches'}{$patch} = $RPS{'Post Upgrade Patches'}{$patch}; } } delete $RPS{'Post Upgrade Patches'}; } printResults(); } # End checkPatches ### # checkPatchesInstalling: Checks if any patches are installing sub checkPatchesInstalling { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal (qq[ 2>&1 ${SUDO} isi_gather_cluster_info -p | ]. qq[ awk "/'installing'/,/\]/ {print}" ]); } else { @gather = gatherLocal (q[ 2>&1 awk '//,/<\/installing>/ {print}' ]. q[ local/node_info_cache.tar/ifs/_ifsvar/db/node_info_cache/*/*.xml ]); } # Gather a list of patches that are installing my %patches_installing = (); foreach my $line (@gather) { my @patches = ($line =~ /'([^']+)'/g); next unless @patches; foreach my $patch (@patches) { next if $patch eq "installing"; $patches_installing{$patch} = 1; } } # Flag if any patches are found as installing fail("The following patches were found as installing: ". join(", ", sort keys %patches_installing)) if %patches_installing; } # End checkPatchesInstalling sub checkPatchHC8007 { local $patch = "HealthCheck_8.0.0.7_v2.0"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @deprecated_patches = hasPatch("HealthCheck_8.0.0.7_v1.0"); local $impacts = "During installation or removal of $patch, the platform API service (isi_papi_d) will be restarted along with the healthcheck service."; checkPatch(); } sub checkPatchHC810 { local $patch = "HealthCheck_8.1.0_v2.0"; local $re_versions = qr/^(8\.1\.0\.[24])$/; local @deprecated_patches = hasPatch("HealthCheck_8.1.0_v1.0"); local $impacts = "During installation or removal of $patch, the platform API service (isi_papi_d) will be restarted along with the healthcheck service."; checkPatch(); } sub checkPatchHC812 { local $patch = "HealthCheck_8.1.2_2019-01"; local $re_versions = qr/^(8\.1\.2\.[01])$/; local @deprecated_patches = hasPatch("HealthCheck_8.1.2_(v1.0|v2.0)"); local $impacts = "During installation or removal of $patch, the platform API service (isi_papi_d) will be restarted along with the healthcheck service."; checkPatch(); } ### # gatherPatches: Gathers a list of installed patches. sub gatherPatches { return if wasCalled(); gatherPatchesLocal8(); gatherPatchesLocal7(); } # End gatherPatches ### # gatherPatchesLocal7: Gathers a list of installed patches. sub gatherPatchesLocal7 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") >= 0) { debug("gatherPatchesLocal7: Skipping. OneFS version 8.0 or later"); return; } # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array "isi pkg info --forced_local | grep '^[[:alnum:]]'"]); } else { %gather = gatherCluster(qq[2>&1 grep '^[[:alnum:]]' */packages_local]); } # Assign CLUSTER_INFO based on gather my %patches = (); foreach my $lnn (@LNNS) { foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /^\s*(\S+):/; push(@{$patches{$1}}, $lnn); } } foreach my $patch (keys %patches) { $CLUSTER_INFO{'Patches'}{$patch} = compressRange(@{$patches{$patch}}); debug("CLUSTER_INFO=>Patches=>${patch}=$CLUSTER_INFO{'Patches'}{$patch}"); } } # End gatherPatchesLocal7 ### # gatherPatchesLocal8: Gathers a list of installed patches. sub gatherPatchesLocal8 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") < 0) { debug("gatherPatchesLocal8: Skipped, version prior to OneFS 8"); return; } # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi upgrade patches list -v --local | grep 'Patch Name']); } else { %gather = gatherCluster(qq[2>&1 grep 'Patch Name' */patch_list_local]); } # Assign CLUSTER_INFO based on gather my %patches = (); foreach my $lnn (@LNNS) { foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /^\s*Patch Name: (\S+)/; push(@{$patches{$1}}, $lnn); } } foreach my $patch (keys %patches) { $CLUSTER_INFO{'Patches'}{$patch} = compressRange(@{$patches{$patch}}); debug("CLUSTER_INFO=>Patches=>${patch}=$CLUSTER_INFO{'Patches'}{$patch}"); } } # End gatherPatchesLocal8 ### # hasPatch: Checks if the cluster has installed a patch that matches the regex provided sub hasPatch { # Gather patches gatherPatches(); # Return an empty list if there are no patches return () unless $CLUSTER_INFO{'Patches'}; # Return all patches is there is no regex string return sort keys %{$CLUSTER_INFO{'Patches'}} unless @_; # Return patches that match the provided regex pattern return grep(/$_[0]/, sort keys %{$CLUSTER_INFO{'Patches'}}); } # End hasPatch $CHECKS{'checkAspera'}{'Description'} = "Checks for aspera services being enabled, indicating that Aspera is in use on the cluster."; sub checkAspera { return if wasCalled(); printCheck("Aspera"); gatherServices(); if (($CLUSTER_INFO{'Services'}{'asperacentral'} and $CLUSTER_INFO{'Services'}{'asperacentral'} eq "Enabled") or ($CLUSTER_INFO{'Services'}{'asperanoded'} and $CLUSTER_INFO{'Services'}{'asperanoded'} eq "Enabled")) { warning("If you are running Aspera for Isilon, you must reinstall Aspera after you upgrade to restore the Aspera service. Please refer to the OneFS Upgrade Planning and Process Guide."); } addCustomerInformation($_) foreach (@WARN, @FAIL); info("Refer to KB210577 (https://support.emc.com/kb/210577) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkAspera ### # checkAuthStatus: Checks authentication status on each node. # Warns if any authentication providers are not online or active. $CHECKS{'checkAuthStatus'}{'Description'} = "Checks authentication status on each node. Warns if any authentication providers are not online or active. Checks for RFC2307 and Auto-locate of GID/UIDs and points to KB335338"; sub checkAuthStatus { return if wasCalled(); printCheck("Authentication Status"); gatherServices(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.0") >= 0) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi auth status --verbose --format list"); } else { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi auth status --verbose"); } } else { %gather = gatherCluster("2>&1 grep -H . */isi_auth_ads_status */isi_auth"); } # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } my $id = ""; foreach my $line (@{$gather{$lnn}}) { if ($line =~ /^\s*Domain:.*?(\S+)/ or $line =~ /^\s*ID:.*?(\S+)/) { $id = $1; } next unless $id; if ($line =~ /^\s*Status:.*?(\S+)/) { my $status = $1; verbose("Provider:$id Status:$status"); warning("On node $lnn, authentication provider $id is $status") if ($status !~ /active|online|disabled/); $id = ""; } } } checkRFC2307(); info("Refer to KB210520 (https://support.emc.com/kb/210520) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkAuthStatus $CHECKS{'checkCapacity'}{'Description'} = "Verifies cluster capacity based on documented numbers the Upgrade Planning and Progress Guide. Warns if approaching."; sub checkCapacity { return if wasCalled(); printCheck("Cluster Capacity"); gatherIsiStatPools(); gatherStat(); verboseHeader("Cluster Capacity"); verbose("Cluster VHS Size: $CLUSTER_INFO{'VHS Size'}") if $CLUSTER_INFO{'VHS Size'}; verbose("Cluster HDD Storage: $CLUSTER_INFO{'HDD Storage'}") if $CLUSTER_INFO{'HDD Storage'}; verbose("Cluster HDD Used: $CLUSTER_INFO{'HDD Used'}") if $CLUSTER_INFO{'HDD Used'}; verbose("Cluster SSD Storage: $CLUSTER_INFO{'SSD Storage'}") if $CLUSTER_INFO{'SSD Storage'}; verbose("Cluster SSD Used: $CLUSTER_INFO{'SSD Used'}") if $CLUSTER_INFO{'SSD Used'}; my @pools_over_90_percent = (); my @pools_over_88_percent = (); my @all_pools = (); debug("checkCapacity: Checking pool usages"); if ($CLUSTER_INFO{'Pools'}) { foreach my $name (keys %{$CLUSTER_INFO{'Pools'}}) { my %pool = %{$CLUSTER_INFO{'Pools'}{$name}}; verbose("Pool $name Health: $pool{'Health'}") if $pool{'Health'}; verbose("Pool $name Protection: $pool{'Protection'}") if $pool{'Protection'}; verbose("Pool $name VHS Size: $pool{'VHS'}") if $pool{'VHS'}; verbose("Pool $name Percent Used: $pool{'Percent Used'}%") if $pool{'Percent Used'}; verbose("Pool $name HDD Storage: $pool{'HDD Storage'}") if $pool{'HDD Storage'}; verbose("Pool $name SSD Storage: $pool{'SSD Storage'}") if $pool{'SSD Storage'}; next unless $pool{'Percent Used'}; push(@all_pools, $name); if ($pool{'Percent Used'} >= 90) { push(@pools_over_90_percent, $name); debug("checkCapacity: !!! Pool $name is over 90% used"); } elsif ($pool{'Percent Used'} >= 88) { push(@pools_over_88_percent, $name); } } } if (@pools_over_90_percent and (scalar(@pools_over_90_percent) eq scalar(@all_pools))) { fail("All pools above 90% capacity"); } if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { foreach my $pool (@pools_over_90_percent) { fail("Pool $pool at $CLUSTER_INFO{'Pools'}{$pool}{'Percent Used'}% capacity"); } } else { foreach my $pool (@pools_over_90_percent) { warning("Pool $pool at $CLUSTER_INFO{'Pools'}{$pool}{'Percent Used'}% capacity"); } } foreach my $pool (@pools_over_88_percent) { warning("Pool $pool at $CLUSTER_INFO{'Pools'}{$pool}{'Percent Used'}% capacity"); } # Fail if total cluster usage is over 90%, warn if approaching if ($CLUSTER_INFO{'HDD Percent Used'} and $CLUSTER_INFO{'HDD Percent Used'} >= 90) { fail("Cluster is $CLUSTER_INFO{'HDD Percent Used'}% used");#, recommended minimum of available space on the cluster is 10%."); } elsif ($CLUSTER_INFO{'HDD Percent Used'} and $CLUSTER_INFO{'HDD Percent Used'} >= 88) { warning("Cluster is $CLUSTER_INFO{'HDD Percent Used'}% used");#, this is approaching the recommended minimum of available space on the cluster of 10%."); } # Check usage on each node foreach my $lnn (@LNNS) { if ($NODE_INFO{$lnn}{'SSD Storage'} and $NODE_INFO{$lnn}{'SSD Storage'} =~ /(\d+)%/) { if ($1 >= 92) { fail("Node $lnn SSD Capacity is at ${1}%"); } elsif ($1 >= 88) { warning("Node $lnn SSD Capacity is at ${1}%"); } } next unless $NODE_INFO{$lnn}{'Percent Used'}; verbose("Node $lnn is at $NODE_INFO{$lnn}{'Percent Used'}% capacity"); if ($NODE_INFO{$lnn}{'Percent Used'} >= 92) { fail("Node $lnn is at $NODE_INFO{$lnn}{'Percent Used'}% capacity");#, recommended minimum amount of available space for each node is 8%."); } elsif ($NODE_INFO{$lnn}{'Percent Used'} >= 88) { warning("Node $lnn is at $NODE_INFO{$lnn}{'Percent Used'}% capacity");#, this is approaching the recommended minimum amount of available space for each node of 8%."); } } addCustomerInformation("Per the 'Check for free node space' section of the Upgrade Planning and Process Guide (https://support.emc.com/docu50235): The total amount of used space on the entire cluster must not exceed 90 percent full, and the total amount of used space on each node must not exceed 92 percent full. Do not continue with the OneFS upgrade if the minimum available-space requirement for the cluster is not met. For more information, see the Best Practices Guide for Maintaining Enough Free Space on Isilon Clusters and Pools (https://support.emc.com/docu48119).") if (@WARN or @FAIL); info("Refer to KB210504 (https://support.emc.com/kb/210504) for details.") if (@FAIL or @WARN); printResults(); } # End checkCapacity $CHECKS{'checkContact'}{'Description'} = "Checks contact information configured in CELog (only useful when called with --extra)"; sub checkContact { return if wasCalled(); printCheck("Contact Information"); gatherLicense(); verboseHeader("Contact Information"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig | grep contact_info]); } elsif (-e "local/isi_gconfig") { @gather = gatherLocal(qq[2>&1 grep contact_info local/isi_gconfig]); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { @gather = gatherLocal(qq[2>&1 grep contact_info local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]); } # Analyze information for issues. my $secondary_name = ""; foreach my $line (@gather) { if ($line =~ /celog.contact_info.company.*?= \"?(.*?)\"?$/) { verbose("Company Name: $1"); } elsif ($line =~ /celog.contact_info.location.*?= \"?(.*?)\"?$/) { verbose("Location: $1"); } elsif ($line =~ /celog.contact_info.primary_name.*?= \"?(.*?)\"?$/) { verbose("Primary Name: $1"); } elsif ($line =~ /celog.contact_info.primary_phone1.*?= \"?(.*?)\"?$/) { verbose("Primary Phone: $1"); } elsif ($line =~ /celog.contact_info.primary_email.*?= \"?(.*?)\"?$/) { verbose("Primary E-mail: $1"); } elsif ($line =~ /celog.contact_info.secondary_name.*?= \"?(.*?)\"?$/) { verbose("Secondary Name: $1"); $secondary_name = $1; } elsif ($line =~ /celog.contact_info.secondary_phone1.*?= \"?(.*?)\"?$/) { verbose("Secondary Phone: $1"); } elsif ($line =~ /celog.contact_info.secondary_email.*?= \"?(.*?)\"?$/) { verbose("Secondary E-mail: $1"); } } if ($CLUSTER_INFO{'OneFS License ID'} and $CLUSTER_INFO{'OneFS'} =~ /^8\.1\.1\.0$|^8\.1\.0\.[012]$/ and $secondary_name !~ /$CLUSTER_INFO{'OneFS License ID'}/) { warning("Isilon SWID not found in the Secondary Name field, see KB517765 for instructions on how to add it"); } printResults(); } # End checkContact $CHECKS{'checkEmailSettings'}{'Description'} = "Checks Email settings configured in CELog (only useful when called with --extra)"; sub checkEmailSettings { return if wasCalled(); printCheck("Email Settings"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig | grep "celog_settings" ]); } elsif (-e "local/isi_gconfig") { @gather = gatherLocal(qq[2>&1 grep "celog_settings" local/isi_gconfig]); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { @gather = gatherLocal(qq[2>&1 grep "celog_settings" local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]); } verboseHeader("Email Settings"); # Analyze information for issues. foreach my $line (@gather) { if ($line =~ /celog\.celog_settings\.mail_relay.*?= \"?(.*?)\"?$/) { verbose("SMTP Relay Address: $1"); } elsif ($line =~ /celog\.celog_settings\.smtp_port.*?= \"?(.*?)\"?$/) { verbose("SMTP Relay Port: $1"); } elsif ($line =~ /celog\.celog_settings\.mail_sender.*?= \"?(.*?)\"?$/) { verbose("Send E-mail as: $1"); } elsif ($line =~ /celog\.celog_settings\.mail_subject.*?= \"?(.*?)\"?$/) { verbose("Subject Line: $1"); } elsif ($line =~ /celog\.celog_settings\.use_smtp_auth.*?= \"?(.*?)\"?$/) { verbose("Use SMTP Authentication: $1"); } } printResults(); } # End checkEmailSettings ### # checkEncoding: Checks exports and cluster configuration for # if utf-8/default encoding. $CHECKS{'checkEncoding'}{'Description'} = "Checks exports and cluster configuration for if utf-8/default encoding."; sub checkEncoding { return if wasCalled(); printCheck("Cluster Encoding"); verboseHeader("Cluster Encoding"); gatherArrayXML(); gatherNFSExport(); verbose("Cluster Encoding: $CLUSTER_INFO{'Encoding'}") if $CLUSTER_INFO{'Encoding'}; my @ids = (); foreach my $ID (sort {$a <=> $b} keys %{$CLUSTER_INFO{'NFS Export'}}) { next unless ($CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'}); verbose("NFS Export ID $ID Encoding: $CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'}"); push(@ids, $ID) if ($CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'} !~ /DEFAULT|UTF-8/i); } if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") < 0 and $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.2") >= 0 and compareVersions($RPS{'Target'}, "8.1.1") < 0) { if ($CLUSTER_INFO{'Encoding'} and $CLUSTER_INFO{'Encoding'} !~ /utf-8/) { fail("ETA483840: The cluster is using $CLUSTER_INFO{'Encoding'} encoding and should not be upgraded to 7.2 or later."); } if (@ids) { fail("ETA483840: The following NFS Export IDs: ".join(",",@ids)." use non-DEFAULT/UTF-8 encoding and should not be upgraded to 7.2 or later."); } } addCustomerInformation($_) foreach (@WARN, @FAIL); info("Refer to KB210627 (https://support.emc.com/kb/210627) for details.") if (@WARN or @FAIL); printResults($CLUSTER_INFO{'Encoding'}); } # End checkEncoding $CHECKS{'checkEvents'}{'Description'} = "Checks events on all the nodes, failure if any critical events exist."; sub checkEvents { return if wasCalled(); printCheck("Critical Events"); gatherEvents(); # Analyze information for issues. foreach my $id (sort keys %{$CLUSTER_INFO{'Events'}}) { next unless $CLUSTER_INFO{'Events'}{$id}{'Severity'} and $CLUSTER_INFO{'Events'}{$id}{'Ended'} and $CLUSTER_INFO{'Events'}{$id}{'Lnn'} and $CLUSTER_INFO{'Events'}{$id}{'Message'}; if ($CLUSTER_INFO{'Events'}{$id}{'Severity'} =~ /^c/i and $CLUSTER_INFO{'Events'}{$id}{'Ended'} eq "--") { fail("Critical event $id for node ".join(",", @{$CLUSTER_INFO{'Events'}{$id}{'Lnn'}}).": ".join(",", @{$CLUSTER_INFO{'Events'}{$id}{'Message'}})); } } info("Refer to KB210506 (https://support.emc.com/kb/210506) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkEvents $CHECKS{'checkFileSharing'}{'Description'} = "Checks if Atime is enabled"; sub checkFileSharing { return if wasCalled(); printCheck("File Sharing"); gatherATime(); if ($CLUSTER_INFO{'ATime'}{'Enabled'}) { debug("checkFileSharing: Atime is enabled"); if ($CLUSTER_INFO{'ATime'}{'Grace Period'}) { if ($CLUSTER_INFO{'ATime'}{'Grace Period'} < 3600000) { critical("Atime Enabled and grace period less than one hour, see KB16764 for details on Atime."); } elsif ($CLUSTER_INFO{'ATime'}{'Grace Period'} < 86400000) { warning("Atime Enabled and grace period less than one day, see KB16764 for details on Atime."); } } } # Output Check result printResults(); } # End checkFileSharing $CHECKS{'checkHDFS'}{'Description'} = "Display HDFS details, only useful when run with --extra"; sub checkHDFS { return if wasCalled(); printCheck("HDFS"); gatherLicense(); verboseHeader("HDFS"); verbose("HDFS License Status: $CLUSTER_INFO{'License'}{'HDFS'}{'Status'}") if $CLUSTER_INFO{'License'}{'HDFS'}{'Status'}; verbose("HDFS License Expiration: $CLUSTER_INFO{'License'}{'HDFS'}{'Expiration'}") if $CLUSTER_INFO{'License'}{'HDFS'}{'Expiration'}; my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi_gconfig -t hdfs"); } elsif ( -e 'local/isi_gconfig_hdfs') { @gather = gatherLocal("2>&1 cat local/isi_gconfig_hdfs"); } elsif ( -e 'local/ifsvar_modules.tar/modules/hdfs_d/config/policy.gc') { @gather = gatherLocal("2>&1 cat local/ifsvar_modules.tar/modules/hdfs_d/config/policy.gc"); } my %lg_bln_sz = ( 12 => '4K', 13 => '8K', 14 => '16K', 15 => '32K', 16 => '64K', 17 => '128K', 18 => '256K', 19 => '512K', 20 => '1M', 21 => '2M', 22 => '4M', 23=> '8M', 24=> '16M', 25=> '32M', 26=> '64M', 27=> '128M', 28=> '256M', 29=> '512M', 30=> '1G' ); my %dec_checksum_type = ( 0 => 'none', 1 => 'crc32', 2 => 'crc32c' ); my %server_log_level = ( 0 => 'emerg', 1 => 'alert', 2 => 'crit', 3 => 'err', 4 => '???', 5 => 'notice', 6 => 'info', 7 => 'debug' ); foreach my $line (@gather) { if ($line =~ /lg_block_sz.*=\s+(\d+)/) { verbose("HDFS Default Block Size: $lg_bln_sz{$1}"); } if ($line =~ /declared_checksum_type.*=\s+(\d+)/) { verbose("HDFS Default Checksum Type: $dec_checksum_type{$1}"); } if ($line =~ /log_level.*=\s+(\d+)/) { verbose("HDFS Server Log Level: $server_log_level{$1}"); } if ($line =~ /num_threads.*=\s+(\d+)/) { verbose("HDFS Server Threads: $1"); } } # Output Check result printResults(); } # End checkHDFS ### # checkHealth: Verifies cluster health status and node health status. $CHECKS{'checkHealth'}{'Description'} = "Verifies cluster health status and node health status."; sub checkHealth { return if wasCalled(); printCheck("Cluster Health Status"); gatherStatus(); gatherIsiStatPools(); gatherStat(); # Fail if cluster health is not OK if ($CLUSTER_INFO{'Health'} and $CLUSTER_INFO{'Health'} ne "OK") { fail("The cluster health is $CLUSTER_INFO{'Health'}"); } # Check each node for issues foreach my $node (sort {$a <=> $b} keys %NODE_INFO) { next if !$NODE_INFO{$node}{'Health'}; if ($NODE_INFO{$node}{'Health'} =~ /SMARTFAIL/) { # Fail if cluster is reporting SMARTFAIL fail("Node $node is reporting SMARTFAIL"); } else { if ($NODE_INFO{$node}{'Health'} =~ /.A../) { # Fail if cluster is reporting ATTENTION fail("Node $node is reporting ATTENTION"); } if ($NODE_INFO{$node}{'Health'} =~ /D.../) { # Fail if cluster is reporting DOWN fail("Node $node is reporting DOWN"); } if ($NODE_INFO{$node}{'Health'} =~ /..S./) { # Fail if cluster is reporting SMARTFAIL fail("Node $node is reporting SMARTFAIL"); } if ($NODE_INFO{$node}{'Health'} =~ /...R/) { # Fail if cluster is reporting READ-ONLY fail("Node $node is reporting READ-ONLY"); } } } info("Refer to KB210505 (https://support.emc.com/kb/210505) for details.") if (@FAIL or @WARN); printResults(); } # End checkHealth $CHECKS{'checkIDI'}{'Description'} = "Checks for IDI errors in the past 90 days"; sub checkIDI { return if wasCalled(); printCheck("IDI Errors"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array cat /var/log/idi.log 2>&1]); } else { %gather = gatherCluster(qq[2>&1 grep -H . */varlog.tar/log/idi.log 2>&1]); } # Analyze information for issues. foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { my $idi_error_count = 0; foreach my $line (@{$gather{$lnn}}) { # Gather timestamp of IDI error next unless ($line =~ /^\s*(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/); my $timestamp = timelocal($6, $5, $4, $3, $2 - 1, $1); # Ignore if older than 90 days next if $RPS{'Runtime'} - $timestamp > 7776000; # Gather IDI errors if ($line =~ /kernel:.*?(ENOTC\|Stack dump|idi_shallow_verify_failure|CRC|bhv|mmb|ffb|DSR failure)/) { $idi_error_count++; debug("CheckIDI: $line") } } # Review gathered IDI error information if ($idi_error_count > 0) { critical("Detected $idi_error_count IDI errors on node $lnn in the last 90 days"); } } # Output Check result printResults(); } # End checkIDI ### # checkISCSI: Checks for iSCSI LUNs being configured in /ifs/.ifsvar/iscsi/iscsi.conf $CHECKS{'checkISCSI'}{'Description'} = "Checks for iSCSI LUNs being configured in /ifs/.ifsvar/iscsi/iscsi.conf (OneFS prior to 8.x only)"; sub checkISCSI { return if wasCalled(); return if compareVersions($CLUSTER_INFO{'OneFS'}, "8") >= 0; return if !$RPS{'Target'} or compareVersions($RPS{'Target'}, "8") < 0; checkLicense(); printCheck("iSCSI LUNs"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 grep -n ^LUN /ifs/.ifsvar/iscsi/iscsi.conf"); } else { @gather = gatherLocal("2>&1 grep -n ^LUN local/ifsvar_iscsi.tar/iscsi/iscsi.conf"); } # Analyze information for issues. my @iscsiluns = (); foreach my $line (@gather) { next unless $line =~ /^\d+:LUN/; next if $line =~ / -H/; push(@iscsiluns, $line); } foreach my $iscsilun (@iscsiluns) { fail("found active iSCSI LUN in line $iscsilun"); } info("The iSCSI module is removed in OneFS 8.x and later. To avoid issues with data associated with iSCSI LUNs following an upgrade, the LUNs must be inactive or removed prior to completing the upgrade checks. For details see https://support.emc.com/kb/494931") if @FAIL; # Output Check result printResults(); } # End checkISCSI $CHECKS{'checkJobHistory'}{'Description'} = "Checks job history for issues, currently just MediaScan issues."; sub checkJobHistory { return if wasCalled(); printCheck("Jobs History"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'grep -h -E "MediaScan.*Succeeded\$" /var/log/messages']); } else { %gather = gatherCluster(qq[2>&1 grep -H -E "MediaScan.*Succeeded\$" */varlog.tar/log/messages]); } # Analyze information for issues. my $most_recent_mediascan = 0; foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $gather{$lnn}; foreach my $line (@{$gather{$lnn}}) { debug("checkJobHistory: Node $lnn: $line"); if ($line =~ /\s*(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/) { my $timestamp = timelocal($6, $5, $4, $3, $2 - 1, $1); debug("checkJobHistory: $timestamp"); $most_recent_mediascan = $timestamp if $timestamp > $most_recent_mediascan; } } } if (!$most_recent_mediascan) { warning("No MediaScan job history found"); } elsif ($RPS{'Timestamp'} - $most_recent_mediascan > 90 * 24 * 60 * 60) { warning("Media scan not completed in last 90 days") } elsif ($RPS{'Timestamp'} - $most_recent_mediascan > 60 * 24 * 60 * 60) { info("Media scan not completed in last 60 days") } printResults(); } # End checkJobHistory ### # checkJobStatus: Checks for running jobs that would impact an upgrade. $CHECKS{'checkJobStatus'}{'Description'} = "Checks for running jobs that would impact an upgrade."; sub checkJobStatus { return if wasCalled(); printCheck("Jobs Status"); gatherJobStatus(); foreach my $state (%{$CLUSTER_INFO{'Jobs'}}) { foreach my $type (@{$CLUSTER_INFO{'Jobs'}{$state}}) { if ($type =~ /Upgrade/i) { if ($RPS{'Target'}) { fail("$type is currently $state and needs to complete prior to an upgrade to OneFS."); } else { warning("$type is currently $state and needs to complete prior to an upgrade to OneFS."); } } elsif ($type =~ /IntegrityScan|FlexProtect/i) { fail("$type is currently $state and needs to complete prior to any upgrade activities."); } elsif ($RPS{'Target'} and $type =~ /dedupe/i and compareVersions($RPS{'Target'}, "8.0") >= 0) { critical("$type is currently $state and needs to be canceled prior to an upgrade to OneFS."); } } } if ($RPS{'Target'} and (compareVersions($RPS{'Target'}, "7.1.1.5") < 0 or $RPS{'Target'} =~ /^7\.2\.0\.[012]$/)) { warning("MediaScan fixes available but not in $RPS{'Target'}."); } if (!$RPS{'Target'} and (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1.1.5") < 0 or $CLUSTER_INFO{'OneFS'} =~ /^7\.2\.0\.[012]$/)) { warning("MediaScan fixes available with OneFS upgrade"); } info("Refer to KB210522 (https://support.emc.com/kb/210522) for details.") if (@FAIL or @WARN); printResults(); } # End checkJobStatus ### # checkKB495911: Checks for an issue where having a large number of # SyncIQ report files causes a tar process to delay other upgrade # processes potentially leaving the cluster in a temporary DU scenario # for an extended period of time. sub checkKB495911 { return if wasCalled(); gatherSyncIQ(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'} and -e '/ifs/.ifsvar/modules/tsm/sched/reports/reports.db') { @gather = gatherLocal("2>&1 sqlite3 /ifs/.ifsvar/modules/tsm/sched/reports/reports.db 'select policy_id, count(policy_id) from reports group by policy_id'"); } elsif (-e 'local/ifsvar_modules.tar/modules/tsm/sched/reports/reports.db') { @gather = gatherLocal("2>&1 sqlite3 local/ifsvar_modules.tar/modules/tsm/sched/reports/reports.db 'select policy_id, count(policy_id) from reports group by policy_id'"); } # Analyze information for issues. my $total_estimated_files_count = 0; my %policies_with_report_issues = (); verboseHeader("SyncIQ Reports"); verbose(sprintf("%-32s %-20s %-20s %s", "Policy ID", "Policy Name", "Report Max Count", "Estimated Count")); foreach my $line (@gather) { next unless $line =~ /\|/; my ($pid, $estimate) = split(/\|/, $line); $total_estimated_files_count += $estimate; next unless $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Max Count'} and $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Name'}; verbose(sprintf("%-32s %-30s %-20s %s", $pid, $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Name'}, $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Max Count'}, $estimate)); next unless ($estimate > 2000 and $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Max Count'} * 2 < $estimate); $policies_with_report_issues{$pid} = $estimate; } verbose("Total estimated files count: $total_estimated_files_count"); # Flag an issue where during an OneFS upgrade, the process of backing up SyncIQ files will delay other services leading to an extended period of DU. if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0 and $RPS{'Target'} and $RPS{'Target'} =~ /^8\.0\.0\.[012345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/) { if ($total_estimated_files_count > 100000) { addCustomerInformation(critical("The estimated total number of SyncIQ report files is $total_estimated_files_count, this may extend the duration of cluster unavailability during an upgrade to OneFS $RPS{'Target'}. A fix for this is available in OneFS 8.0.0.6, OneFS 8.0.1.2, and OneFS 8.1.0.1 and later OneFS versions.")); } if (%policies_with_report_issues) { fail("The below listed policies have estimated more than double the amount of SyncIQ reports they should have."); fail(sprintf("%-30s %-20s %s", "Policy Name", "Report Max Count", "Estimated Count")); foreach my $pid (keys %policies_with_report_issues) { fail(sprintf("%-30s %-20s %s", $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Name'}, $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Max Count'}, $policies_with_report_issues{$pid})); } } } elsif (%policies_with_report_issues) { warning("The below listed policies have estimated more than double the amount of SyncIQ reports they should have."); warning(sprintf("%-30s %-20s %s", "Policy Name", "Report Max Count", "Estimated Count")); foreach my $pid (keys %policies_with_report_issues) { warning(sprintf("%-30s %-20s %s", $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Name'}, $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{'Max Count'}, $policies_with_report_issues{$pid})); } } } # End checkKB495911 ### # checkKB496582: Checks for auth rules issues detailed in KB496582. $CHECKS{'checkKB496582'}{'Description'} = "Checks for auth rules issues detailed in KB496582."; sub checkKB496582 { return if wasCalled(); return if compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.4") >= 0; return unless $RPS{'Target'} and compareVersions($RPS{'Target'}, "8.0.0.4") >= 0; printCheck("KB496582"); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi_gconfig | grep MappingRules"); } elsif (-e "local/isi_gconfig") { @gather = gatherLocal("2>&1 grep MappingRules local/isi_gconfig"); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { @gather = gatherLocal("2>&1 grep MappingRules local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } my %rules = (); foreach my $line (@gather) { next unless ($line =~ /Zones\.([^\.]+)\.MappingRules [^\[]+\[ (.+) \]/); $rules{$1} = $2; debug("checkKB496582: Zone: $1 Rules: $rules{$1}\n"); } my $KB496582 = 0; foreach my $zone (keys %rules) { foreach my $rule (parseRules($rules{$zone})) { $KB496582 = 1 if ($rule->{'options'} and $rule->{'options'} =~ /group/ and $rule->{'options'} !~ /groups/); } } if ($KB496582) { warning("Customer should review KB 496582 prior to the OneFS upgrade, a User Mapping Rule was found that may be impacted by the change of behavior in OneFS 8.0.0.4."); addCustomerInformation("Review KB 496582 prior to the OneFS upgrade, a User Mapping Rule was found that may be impacted by the change of behavior in OneFS 8.0.0.4."); } printResults(); } # End checkKB496582 sub checkKB501267 { return if wasCalled(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array -s sysctl efs.bam.disk_pool_db|grep generation]); } else { %gather = gatherCluster(qq[2>&1 grep -H generation */efs.bam.disk_pool_db]); } # Analyze information for issues. my %generations = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next if (!$gather{$lnn}); foreach my $line (@{$gather{$lnn}}) { if ($line =~ /generation=(\d+)/) { push(@{$generations{$1}}, $lnn); } } } if (scalar(keys %generations) > 1) { warning("Multiple generations observed in Disk Pools DB"); foreach my $generation (sort keys %generations) { warning("generation=$generation on nodes: ".compressRange(@{$generations{$generation}})); } warning("See KB501267 for workaround details"); } } # End checkKB501267 $CHECKS{'checkKB507031'}{'Description'} = "Checks for criteria outlined in KB507031"; sub checkKB507031 { return if wasCalled(); printCheck("KB507031"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array grep -e 'Match' -e 'Ciphers' /etc/mcp/templates/sshd_config]); } else { %gather = gatherCluster(qq[2>&1 grep -H -e 'Match' -e 'Ciphers' */etc.tar/etc/mcp/templates/sshd_config]); } # Analyze information for issues. my @ciphers_after_mb = (); my @mb_without_ciphers = (); my @commented_mb_with_ciphers = (); foreach my $lnn (@LNNS) { next unless $gather{$lnn}; my $has_mb = 0; my $has_ciphers = 0; my $has_commented_mb = 0; foreach my $line (@{$gather{$lnn}}) { if ($line =~ /^\s*Match/) { $has_mb = 1; } if ($line =~ /^\s*Ciphers/) { $has_ciphers = 1; if ($has_mb) { my @results = grep {$_ eq $lnn} @ciphers_after_mb; push(@ciphers_after_mb, $lnn) unless @results; } } if ($line =~ /^\s*#MB/) { $has_commented_mb = 1; } } push(@mb_without_ciphers, $lnn) if $has_mb and !$has_ciphers; push(@commented_mb_with_ciphers, $lnn) if $has_ciphers and $has_commented_mb; } if (@mb_without_ciphers and $RPS{'Target'}) { critical("Follow the 'Preventative Workaround for OneFS upgrades' steps as outlined in KB 507031 when performing the OneFS upgrade to $RPS{'Target'}."); } if (@ciphers_after_mb) { fail("Directive 'Ciphers' is not allowed within a Match block, review KB 507031 for resolution."); } if (@commented_mb_with_ciphers) { fail("Follow the 'Preventative Workaround for OneFS upgrades' post-upgrade steps as outlined in KB 507031."); } # Output Check result printResults(); } # End checkKB507031 $CHECKS{'checkKB519388'}{'Description'} = "Pre-upgrade check for issues outlined in KB519388"; sub checkKB519388 { return if wasCalled(); return unless $RPS{'Target'}; return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0; printCheck("KB519388"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'sysctl efs.snapshot.default_sub_visible']); } else { %gather = gatherCluster(qq[2>&1 grep -H default_sub_visible */efs.snapshot]); } # Analyze information for issues. verboseHeader("KB519388"); verbose(sprintf("%-10s %-10s", "LNN", "efs.snapshot.default_sub_visible")); verbose(sprintf("%-10s %-10s", "-"x10, "-"x10)); my $default_sub_visible = 0; foreach my $lnn (@LNNS) { next unless $gather{$lnn}; foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /efs\.snapshot\.default_sub_visible:\s+(\d+)$/; $default_sub_visible = 1 if $1; verbose(sprintf("%-10s %-10s", $lnn, $1)); } } critical("The sysctl setting efs.snapshot.default_sub_visible is currently enabled", "During an upgrade from OneFS $CLUSTER_INFO{'OneFS'} to $RPS{'Target'} the sysctl setting efs.snapshot.default_sub_visible should be disabled, see KB519388 details on how to disable this setting.", "Take care to re-enable this setting once the upgrade has completed.") if $default_sub_visible; # Output Check result printResults(); } # End checkKB519388 $CHECKS{'checkKB521778'}{'Description'} = "Checks for criteria outlined in KB 521778"; sub checkKB521778 { return if wasCalled(); # Skip unless running or upgrading to impacted versions return unless ($RPS{'Target'} and $RPS{'Target'} =~ /^(8\.0\.0\.[56]|8\.1\.0\.[23])$/) or (!$RPS{'Target'} and $CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.[56]|8\.1\.0\.[23])$/); # Skip unless licensed for SmartQuotas and SyncIQ return unless hasLicense("SmartQuotas"); return unless hasLicense("SyncIQ"); # Skip unless a SyncIQ target gatherSyncIQ(); return unless $CLUSTER_INFO{'SyncIQ'} and $CLUSTER_INFO{'SyncIQ'}{'Target'}; printCheck("KB521778"); gatherPatches(); $RPS{'KB521778'} = 1; warning("This cluster is licensed for SmartQuotas and is a target of SyncIQ. Incoming SyncIQ may fail if there are more than 100 quotas in sub-directories below a SyncIQ target path. See KB 521778 for details on this known issue."); if ($RPS{'Target'}) { if ($RPS{'Target'} =~ /^(8\.0\.0\.5)$/) { critical("Do NOT perform the SyncIQ Rollup Patch 224399 after performing an upgrade to OneFS $RPS{'Target'}."); } elsif ($RPS{'Target'} =~ /^(8\.0\.0\.6)$/) { critical("Do NOT perform the SyncIQ Rollup Patch 215690 or Patch 221638 after performing an upgrade to OneFS $RPS{'Target'}."); } elsif ($RPS{'Target'} =~ /^(8\.1\.0\.2)$/) { critical("Do NOT perform the SyncIQ Rollup Patch 222718 after performing an upgrade to OneFS $RPS{'Target'}."); } elsif ($RPS{'Target'} =~ /^(8\.1\.0\.3)$/) { fail("An upgrade to OneFS $RPS{'Target'} should NOT be performed."); } } if (hasPatch("patch-(224399|215690|221638|222718)") or $CLUSTER_INFO{'OneFS'} =~ /^(8\.1\.0\.3)$/) { if ($RPS{'Target'}) { warning("Cluster current OneFS and patch level put it at risk for KB 521778."); } else { fail("Cluster current OneFS and patch level put it at risk for KB 521778."); } } elsif ($CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.5)$/) { warning("Do NOT perform the SyncIQ Rollup Patch 224399"); } elsif ($CLUSTER_INFO{'OneFS'} =~ /^(8\.0\.0\.6)$/) { warning("Do NOT perform the SyncIQ Rollup Patch 215690 or Patch 221638"); } elsif ($CLUSTER_INFO{'OneFS'} =~ /^(8\.1\.0\.2)$/) { warning("Do NOT perform the SyncIQ Rollup Patch 222718"); } # Output Check result printResults(); } # End checkKB521778 $CHECKS{'checkKB524082'}{'Description'} = "Checks if the cluster is enabled for HTTP clients and flags an issue caused by the change in Apache versions."; sub checkKB524082 { return if wasCalled(); # Skip unless upgrading to a version with Apache 2.4 return unless $RPS{'Target'} and (compareVersions($RPS{'Target'}, "8.1.2.0") >= 0 or ($RPS{'Target'} =~ /^8\.1\.0/ and compareVersions($RPS{'Target'}, "8.1.0.3") >= 0) or ($RPS{'Target'} =~ /^8\.0\.0/ and compareVersions($RPS{'Target'}, "8.0.0.7") >= 0)); # Skip if running a version with Apache 2.4 return if (($CLUSTER_INFO{'OneFS'} =~ /^8\.0\.0/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.7") >= 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8\.1\.0/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.3") >= 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8\.1\.2/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.2.0") >= 0)); printCheck("KB524082"); gatherServices(); # Gather information from cluster or from logset my @gather = (); if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.0") < 0) { if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 grep "httpdata2" /etc/mcp/override/apache2.xml]); } else { @gather = gatherLocal(qq[2>&1 grep "httpdata2" */etc.tar/etc/mcp/override/apache2.xml local/ifsvar_etc.tar/etc/mcp/override/apache2.xml]); } } else { if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi_gconfig -t http-config"); } else { @gather = gatherLocal("2>&1 cat local/ifsvar_modules.tar/modules/http/http_config.gc"); } } # Analyze information for issues. my $enabled_config = 0; foreach my $line (@gather) { $enabled_config = 1 if $line =~ /^redirect.*false/; $enabled_config = 1 if $line =~ /httpdata2.*>enabled= 0; printCheck("KB527312"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 cat /ifs/.ifsvar/modules/smartpools/drive_purposing.db]); } else { @gather = gatherLocal(qq[2>&1 cat local/ifsvar_modules.tar/modules/smartpools/drive_purposing.db]); } # Analyze information for issues. my $affected = 0; foreach my $line (@gather) { # If there are any empty lists they may be affected $affected = 1 if $line =~ /\[\s*\]/; # Once found affected, drop out of loop last if $affected; } # Flag issues detected if ($affected) { fail("Cluster may be affected by KB527312 during an upgrade from $CLUSTER_INFO{'OneFS'} to $RPS{'Target'}."); fail("The malformed drive_purposing.db file should be corrected prior to performing the OneFS upgrade."); } # Output Check result printResults(); } # End checkKB527312 ### # checkLicense: Checks licenses and gives direction based on licensed # features. InsigntIQ and vCenter licenses, gives information from # compatibility guide. iSCSI, instructs to only perform simultaneous # OneFS upgrades and that it is not supported in 8.0. $CHECKS{'checkLicense'}{'Description'} = "Checks licenses and gives direction based on licensed features. InsigntIQ and vCenter licenses, gives information from compatibility guide. iSCSI, instructs to only perform simultaneous OneFS upgrades and that it is not supported in 8.0."; sub checkLicense { return if wasCalled(); printCheck("Licenses"); gatherLicense(); gatherServices(); gatherProductInfo(); if (hasLicense("iSCSI")) { if ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.0") >= 0) { fail("iSCSI is licensed. Support for iSCSI is deprecated as of OneFS 8."); } else { warning("iSCSI is licensed. If there are any iSCSI connections to the cluster, you can only perform a simultaneous upgrade. After all the initiator connections are stopped, set the iSCSI LUN to offline until the upgrade is complete." ); } } if ($RPS{'Target'} and hasLicense("vCenter")) { if (compareVersions($RPS{'Target'}, "7.2") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") < 0) { warning("Isilon for vCenter is licensed. OneFS 7.2 or later does not support Isilon for vCenter plugin."); } if (compareVersions($RPS{'Target'}, "7.0") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.0") < 0) { warning("Isilon for vCenter is licensed. OneFS 7.0 or later supports Isilon for vCenter version 1.3 or later."); } } if ($RPS{'Target'} and (hasLicense("InsightIQ") or compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") >= 0)) { if (($RPS{'Target'} =~ /^8\.1\.1/ and compareVersions($RPS{'Target'}, "8.1.1.1") >= 0) or ($RPS{'Target'} =~ /^8\.1\.0/ and compareVersions($RPS{'Target'}, "8.1.0.3") >= 0) or ($RPS{'Target'} =~ /^8\.0\.0/ and compareVersions($RPS{'Target'}, "8.0.0.7") >= 0)) { if (!(compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.1.1") >= 0 or ($CLUSTER_INFO{'OneFS'} =~ /^8\.1\.0/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.3") >= 0) or ($CLUSTER_INFO{'OneFS'} =~ /^8\.0\.0/ and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.7") >= 0))) { warning("InsightIQ 4.1.2 or later is required for OneFS $RPS{'Target'} support."); } } elsif ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.0") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { warning("InsightIQ 4.0 or later is required for OneFS 8 support."); } elsif ($RPS{'Target'} and compareVersions($RPS{'Target'}, "7.2") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.2") < 0) { warning("InsightIQ 3.1 or later is required for OneFS 7.2 support."); } elsif ($RPS{'Target'} and compareVersions($RPS{'Target'}, "7.0") >= 0 and compareVersions($CLUSTER_INFO{'OneFS'}, "7.0") < 0) { warning("InsightIQ 2.5 or later is required for OneFS 7 support."); } } # Flag major/minor/sub upgrades that do not have a valid signature if ($RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") >= 0 and not $CLUSTER_INFO{'Valid Signature'} and not $CLUSTER_INFO{'Virtual'} and upgradeDigit($CLUSTER_INFO{'OneFS'}, $RPS{'Target'}) <= 3) { fail("License does not have a valid signature. The license must be updated prior to performing the upgrade from $CLUSTER_INFO{'OneFS'} to $RPS{'Target'}."); } # All information above here is in licenses checks should be sent to the customer addCustomerInformation($_) foreach (@WARN, @FAIL); # Flag is iSCSI is disabled but the iSCSI service is enabled as a warning if ((!$RPS{'Target'} or compareVersions($RPS{'Target'}, "8.0") < 0) and !hasLicense("iSCSI") and $CLUSTER_INFO{'Services'}{'isi_iscsi_d'} and $CLUSTER_INFO{'Services'}{'isi_iscsi_d'} eq 'Enabled') { warning("iSCSI license is $CLUSTER_INFO{'License'}{'iSCSI'}{'Status'} and the iSCSI target Daemon service(isi_iscsi_d) is enabled"); } # Flag expired licenses as a warning my @expired_licenses = (); foreach my $name (keys %{$CLUSTER_INFO{'License'}}) { if ($CLUSTER_INFO{'License'}{$name}{'Status'} and $CLUSTER_INFO{'License'}{$name}{'Status'} eq "Expired") { push(@expired_licenses, $name); } } warning("The following licenses have expired: ".join(", ", @expired_licenses)) if @expired_licenses; # Output Check result info("Refer to KB210519 (https://support.emc.com/kb/210519) for details.") if (@WARN or @FAIL); printResults($CLUSTER_INFO{'OneFS License ID'} or ""); } # End checkLicense $CHECKS{'checkListenQueue'}{'Description'} = "Checks for listen queue overflows to be less than 50,000 per node"; sub checkListenQueue { return if wasCalled(); printCheck("Listen Queue Overflows"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array netstat -sn | grep 'listen queue overflow']); } else { %gather = gatherCluster(qq[2>&1 grep -H 'listen queue overflow' */netstat]); } # Analyze information for issues. my @listen_queue = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { next unless ($line =~ /(\d+)\s+listen queue overflow/); verbose("Listen Queue Overflows: $1"); push(@listen_queue, $lnn) if ($1 > 50000); } } warning("Listen Queue Overflows count over 50,000 on the following nodes: ".join(",", @listen_queue)) if @listen_queue; printResults(); } # End checkListenQueue $CHECKS{'checkLWIODLog'}{'Description'} = "Checks /var/log/lwiod.log for known errors occuring in the last 30 days"; sub checkLWIODLog { return if wasCalled(); printCheck("LWIOD Log"); # A list of the last 30 days my @date = (); for (my $i = 0; $i <= 30; $i++) { my ($day, $month, $year) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4,5]; $year += 1900; $month += 1; push(@date, sprintf("%04d-%02d-%02d", $year, $month, $day)); } # A list of the errors to gather my @detect = ('code:0xc0000205', 'GSS-API error calling gss_accept_sec_context:', 'Too many open files', 'Clock skew too great', 'LW_ERROR_CLOCK_SKEW', 'An invalid name was supplied', 'Configuration file does not specify default realm', 'Ticket not yet valid', 'Unspecified GSS failure. Minor code may provide more information', 'Illegal cross-realm ticket', '\.pst', '\.ovt', '\.pap', '\.ost', ); # A list of the errors to ignore my @ignore = ('LW_ERROR_ACCOUNT_DISABLED', 'LW_ERROR_INVALID_PASSWORD', 'LW_ERROR_ACCOUNT_LOCKED', 'LW_ERROR_ACCOUNT_EXPIRED', ); # Compile greps from the lists my $date_grep = join("|", @date); my $detect_grep = join("|", @detect); my $ignore_grep = join("|", @ignore); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'grep -E "^(${date_grep}).*(${detect_grep})" /var/log/lwiod.log | grep -v -E "(${ignore_grep})"']); } else { %gather = gatherCluster(qq[2>&1 grep -E "^(${date_grep}).*(${detect_grep})" */varlog.tar/log/lwiod.log | grep -v -E "(${ignore_grep})"]); } # Analyze information for issues. my $pst_errors = 0; my %error_counts = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless ($gather{$lnn}); debug(@{$gather{$lnn}}); foreach my $line (@{$gather{$lnn}}) { if ($line =~ /\.(pst|ovt|pap|ost)/) { $pst_errors++; next; } elsif ($line =~ /($detect_grep)/) { $error_counts{$1}++; } } } foreach my $error (keys %error_counts) { warning("Found $error_counts{$error} '$error' errors"); } warning("Performance degrading workflow detected, see KB 334538 for details on performance impacts with PST files.") if $pst_errors; printResults(); } # End checkLWIODLog $CHECKS{'checkMessagesLog'}{'Description'} = "Checks /var/log/messages.log for known errors occuring in the last 30 days"; sub checkMessagesLog { return if wasCalled(); printCheck("Messages Log"); # A list of the last 30 days my @date = (); for (my $i = 0; $i <= 30; $i++) { my ($day, $month, $year) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4,5]; $year += 1900; $month += 1; push(@date, sprintf("%04d-%02d-%02d", $year, $month, $day)); } # A list of the error types to gather my @detect = ("bus", "changed linkrate", "dsr", "dwr", "dws", "dwt", "ecc", "error", "fail", "free", "inquiry", "invalid hostname to sm_mon", "isi_flexnet_d", "isi_migrate", "lock", "mca", "memory", "nlm", "non-fatal", "onefs", "panic", "quota", "rbmn", "seconds in syslog", "stack", "stall", "terminated", "unrecovered", "upgrade", "userspace", "watchdog", ); my $awkbegin = "BEGIN{IGNORECASE=1}"; my $awkstring = ""; my $awkend = "END {"; foreach my $i (@detect) { my $key = $i; $key =~ s/[^a-zA-Z0-9]/_/g; $awkstring .= qq[/^[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]].*${i}/{${key}++; ${key}MR=\\\$1}]; $awkend .= qq[if(${key}){print \\\"${key}:\\\" ${key} \\\" MR:\\\" ${key}MR};]; } $awkend .= "}"; # A list of the errors to ignore my @ignore = ('last message repeated'); # Compile greps from the lists my $date_grep = join("|", @date); my $detect_grep = join("|", @detect); my $ignore_grep = join("|", @ignore); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'awk "]. qq[$awkbegin]. qq[(\\\$0 !~ /(${date_grep}).*\\.[01234]>/){next}]. qq[/(${ignore_grep})/{next}]. qq[$awkstring]. qq[$awkend]. qq[" /var/log/messages']); } else { $awkstring =~ s/\\//g; $awkend =~ s/\\//g; %gather = gatherCluster(qq[for i in */varlog.tar/log/messages; do 2>&1]. qq[ grep -H -E '^(${date_grep}).*\.[01234]>' \$i]. qq[|grep -v -E '(${ignore_grep})']. qq[|awk '$awkstring $awkend']. qq[|awk 'BEGIN{FS=":";OFS=":"}{LNN=\$3;\$3=\$2;\$2=\$1;\$1=LNN;print}']. qq[;done]); } # Analyze information for issues. my %error_counts = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless ($gather{$lnn}); debug("--------------------"," Node $lnn ","--------------------"); debug(@{$gather{$lnn}}); foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /^\s*([^:]+):(\d+) MR:(\S+)$/; $error_counts{$lnn}{$1}{'Count'} = $2; $error_counts{$lnn}{$1}{'Most Recent'} = $3; } } foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $error_counts{$lnn}; foreach my $error (keys %{$error_counts{$lnn}}) { warning("Node $lnn: Found $error_counts{$lnn}{$error}{'Count'} '$error' errors. Most recent log entry of this type was: $error_counts{$lnn}{$error}{'Most Recent'}"); } } printResults(); } # End checkMessagesLog $CHECKS{'checkNDMP'}{'Description'} = "Checks for running NDMP sessions"; sub checkNDMP { return if wasCalled(); printCheck("NDMP"); gatherServices(); # Flag to customer that NDMP backups must if ($CLUSTER_INFO{'Services'}{'ndmpd'} and $CLUSTER_INFO{'Services'}{'ndmpd'} =~ /enabled/i) { addCustomerInformation(info("Please ensure that all NDMP backups have completed and are not running ahead of the scheduled upgrade maintenance window to avoid delays.")); addCustomerInformation(info("Make sure to verify compatibility of your NDMP backup application with the destination OneFS of $RPS{'Target'} prior to the scheduled upgrade.")) if $RPS{'Target'}; } # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { @gather = gatherLocal("2>&1 ${SUDO} isi ndmp sessions list"); } else { @gather = gatherLocal("2>&1 ${SUDO} isi ndmp list"); } } # Analyze information for issues. my $active_ndmp = 0; foreach my $line (@gather) { if ($line =~ /^\s*\d.*ACTIVE/ or $line =~ /^\s*\d.* A /) { $active_ndmp = 1; } } critical("There are active NDMP sessions. Prior to any upgrade activity all NDMP sessions must complete or be stopped from the associated backup application.") if $active_ndmp; # Output Check result printResults(); } # End checkNDMP $CHECKS{'checkNetstat'}{'Description'} = "Checks connections counts for specific protocols via netstat"; sub checkNetstat { return if wasCalled(); printCheck("Netstat Connections"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'awk "]. q[\\$4~/\.(20|21)$/{FTP++}]. q[\\$4~/\.(22)$/{SSH++}]. q[\\$4~/\.(135|137|138|139|445)$/{SMB++}]. q[\\$4~/\.(80)$/{HTTP++}]. q[\\$4~/\.(443)$/{HTTPS++}]. q[\\$4~/\.(8080|8081)$/{WEBUI++}]. q[\\$4~/\.(2049)$/{NFS++}]. q[\\$4~/\.(9999|3260|3205|860)$/{ISCSI++}]. q[\\$4~/\.(8021|8020)$/{HDFS++}]. q[END{]. q[print \\"FTP:\\" FTP;]. q[print \\"SSH:\\" SSH;]. q[print \\"SMB:\\" SMB;]. q[print \\"HTTP:\\" HTTP;]. q[print \\"HTTPS:\\" HTTPS;]. q[print \\"WEBUI:\\" WEBUI;]. q[print \\"NFS:\\" NFS;]. q[print \\"ISCSI:\\" ISCSI;]. q[print \\"HDFS:\\" HDFS;]. q[}" <(netstat -lan|grep ESTABLISHED)']); } else { %gather = gatherCluster(qq[for i in */netstat;do echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(20|21)\$/) C++}END{print ":FTP:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(22)\$/) C++}END{print ":SSH:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(135|137|138|139|445)\$/) C++}END{print ":SMB:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(80)\$/) C++}END{print ":HTTP:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(443)\$/) C++}END{print ":HTTPS:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(8080|8081)\$/) C++}END{print ":WEBUI:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(2049)\$/) C++}END{print ":NFS:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(9999|3260|3205|860)\$/) C++}END{print ":ISCSI:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(8021|8020)\$/) C++}END{print ":HDFS:" C}' \$i; done]); } # Analyze information for issues. my %high_connection_counts = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } my %connection_counts = (); foreach my $line (@{$gather{$lnn}}) { next unless ($line =~ /\s*(\S+):(\d+)/); verbose("$1 Client Connections: $2"); $connection_counts{$1} = $2; } push(@{$high_connection_counts{'FTP'}}, $lnn) if ($connection_counts{'FTP'} and $connection_counts{'FTP'} >= 25); push(@{$high_connection_counts{'HDFS'}}, $lnn) if ($connection_counts{'HDFS'} and $connection_counts{'HDFS'} >= 25); push(@{$high_connection_counts{'SMB'}}, $lnn) if ($connection_counts{'SMB'} and $connection_counts{'SMB'} >= 750); push(@{$high_connection_counts{'NFS'}}, $lnn) if ($connection_counts{'NFS'} and $connection_counts{'NFS'} >= 750); push(@{$high_connection_counts{'SSH'}}, $lnn) if ($connection_counts{'SSH'} and $connection_counts{'SSH'} >= 5); push(@{$high_connection_counts{'WEBUI'}}, $lnn) if ($connection_counts{'WEBUI'} and $connection_counts{'WEBUI'} >= 5); push(@{$high_connection_counts{'HTTP'}}, $lnn) if ($connection_counts{'HTTP'} and $connection_counts{'HTTP'} >= 250); push(@{$high_connection_counts{'ISCSI'}}, $lnn) if ($connection_counts{'ISCSI'} and $connection_counts{'ISCSI'} >= 2); push(@{$high_connection_counts{'HTTPS'}}, $lnn) if ($connection_counts{'HTTPS'} and $connection_counts{'HTTPS'} >= 10); } foreach my $protocol (keys %high_connection_counts) { warning("High connection count for $protocol on nodes: ".compressRange(@{$high_connection_counts{$protocol}})); } # Output Check result printResults(); } # End checkNetstat ### # checkNFS: Runs nfsstat and looks for Encountered RPC error $CHECKS{'checkNFS'}{'Description'} = "Runs nfsstat and looks for Encountered RPC error"; sub checkNFS { return if wasCalled(); gatherServices(); printCheck("NFS"); checkNFS8004(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array nfsstat | grep 'Encountered RPC error']); } else { %gather = gatherCluster(qq[2>&1 grep -H 'Encountered RPC error' */nfsstat]); } # Analyze information for issues. my %rpc_errors = (); foreach my $lnn (@LNNS) { foreach my $line (@{$gather{$lnn}}) { $rpc_errors{$lnn} = 1 if ($line =~ /Encountered RPC error/); } } critical("The NFS service is not enabled. If the customer does not use NFS this may be expected and the 'NFS issues detected' may be ignored.") if ($CLUSTER_INFO{'Services'}{'nfs'} and $CLUSTER_INFO{'Services'}{'nfs'} !~ /enabled/i); critical("NFS issues detected on the following nodes: ".compressRange(keys %rpc_errors)) if %rpc_errors; # Output Check result printResults(); } # End checkNFS $CHECKS{'checkOpenFiles'}{'Description'} = "Checks if there are more than 50,000 open files are reported by the kernel in sysctl."; sub checkOpenFiles { return if wasCalled(); printCheck("Kernel Open Files Count"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array sysctl kern.openfiles]); } else { %gather = gatherCluster(qq[2>&1 grep -H . */kern.openfiles]); } # Analyze information for issues. my @over_50k_open_files = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /kern.openfiles:\s+(\d+)$/; verbose("Kernel Open Files Count: $1"); push(@over_50k_open_files, $lnn) if ($1 > 50000); } } if (@over_50k_open_files) { warning("The following nodes were found to have over 50,000 open files: ".join(",", @over_50k_open_files)); info("Refer to KB17281 (https://support.emc.com/kb/17281) for details."); } # Output Check result printResults(); } # End checkOpenFiles ### # checkPartitions: Checks for partition space. # / Cannot be more than 97 percent full. # https://support.emc.com/kb/89552 - Node reached 95% used capacity on the root filesystem. # /ifs Cannot be more than 90 percent full. # https://support.emc.com/kb/167267 - There is at least one smartpool at or over capacity or the SmartPool is near or over capacity. # /var Cannot be more than 90 percent full. # https://support.emc.com/kb/89848 - The /var partition is near capacity (95% used). # /var/crash Cannot be more than 90 percent full. # https://support.emc.com/kb/89823 - The crash partition of a node in the cluster has reached 90% capacity alert. $CHECKS{'checkPartitions'}{'Description'} = "Checks system partition space."; sub checkPartitions { return if wasCalled(); printCheck("System Partition Free Space"); my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array df -Pli"); } else { %gather = gatherCluster("2>&1 grep -H . */df"); } foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } my $mirror_node = 0; my $imdd_node = 0; my $num_blocks = 0.00048828125; foreach my $line (@{$gather{$lnn}}) { verbose($line); $mirror_node = 1 if $line =~ /^\s*\/dev\/mirror\//; $imdd_node = 1 if $line =~ /^\s*\/dev\/imdd/; if ($line =~ /Filesystem\s+(\S+)/) { if ($1 eq "512-blocks") { $num_blocks = 0.00048828125; } elsif ($1 eq "1K-blocks") { $num_blocks = 0.0009765625; } } next unless ($line =~ /\s*(\/.*?)\s+(\d+)\s+(\d+)\s+(-?\d+)\s+(\d+)%\s+(\d+)\s+(\d+)\s+(\d+)%\s+(\S+.*)$/); my $fs = $1; my $total = int($2 * $num_blocks); my $used = int($3 * $num_blocks); my $available = int($4 * $num_blocks); my $capacity = $5; my $iused = $8; my $mount = $9; # Flag root (/) partition over 97% full if ($mount eq "/" and $capacity > 97) { fail("Node $lnn partition $mount is over 97% full."); } # Flag /var partition over 90% full if ($mount eq "/var" and $capacity > 90) { fail("Node $lnn partition $mount is over 90% full."); } # Flag /var/crash partition over 50/90% full if ($mount eq "/var/crash" and $capacity > 90) { fail("Node $lnn partition $mount is over 90% full."); } elsif ($mount eq "/var/crash" and $capacity > 50) { warning("Node $lnn partition $mount is over 50% full."); } # Flag inode usage over 90% if (($mount eq "/" or $mount eq "/var" or $mount eq "/var/crash") and $iused > 97) { fail("Node $lnn partition $mount inode usage is over 97%."); } # Flag var less than 500MB free space for use in node firmware check if ($mount eq "/var" and $available < 500) { push(@{$RPS{'Node Firmware'}{'var'}}, $lnn); } # Node compatibility check for 8.0.0.7 and IQ nodes if ($mount eq "/" and $total < 500) { $RPS{'Root less than 500MB'}{$lnn} = 1; } } if ($mirror_node) { $NODE_INFO{$lnn}{'Bootflash'} = 1; debug("NODE_INFO->${lnn}->Bootflash=1"); push(@{$CLUSTER_INFO{'Bootflash'}}, $lnn); debug("CLUSTER_INFO->Bootflash=$lnn"); } if ($imdd_node) { $NODE_INFO{$lnn}{'IMDD'} = 1; debug("NODE_INFO->${lnn}->IMDD=1"); push(@{$CLUSTER_INFO{'IMDD'}}, $lnn); debug("CLUSTER_INFO->IMDD=$lnn"); } } info("Refer to KB210508 (https://support.emc.com/kb/210508) for details.") if (@WARN or @FAIL); printResults(); } # End checkPartitions $CHECKS{'checkPerformance'}{'Description'} = "Checks cluster performance"; sub checkPerformance { return if wasCalled(); checkHardwareStatus(); printCheck("Performance"); # Pre-defined threshold for performance checks my $THRESHOLD = 80; # A list of the last 7 days my @abbr = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec); my @date = (); for (my $i = 0; $i <= 7; $i++) { my ($day, $month) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4]; push(@date, sprintf("%3s %2s", $abbr[$month], $day)); } my $dategrep = join("|", @date); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array "zgrep -H -E '^Mon|^Tue|^Wed|^Thu|^Fri|^Sat|^Sun|vm.loadavg' /var/log/vmlo* | paste - - | grep -E '$dategrep'"]); } else { %gather = gatherCluster(qq[2>&1 zgrep -H -E '^Mon|^Tue|^Wed|^Thu|^Fri|^Sat|^Sun|vm.loadavg' */varlog.tar/log/vmlo* | paste - - | grep -E '$dategrep']); } # Analyze information for issues. my %stats = (); my %average_above_threshold = (); my ($total_of_averages, $total_of_highest, $node_count); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } # Parse command output foreach my $line (@{$gather{$lnn}}) { next unless $line =~ /vm.loadavg: \{ \S+ \S+ ([0-9.]+) \}/; $stats{$lnn}{'load'} += $1; $stats{$lnn}{'count'}++; if (!$stats{$lnn}{'highest'} or $1 > $stats{$lnn}{'highest'}) { $stats{$lnn}{'highest'} = $1; } } next unless exists $stats{$lnn}{'count'} and exists $stats{$lnn}{'highest'} and exists $stats{$lnn}{'load'}; # Calculate the average load $stats{$lnn}{'average'} = $stats{$lnn}{'load'} / $stats{$lnn}{'count'}; next unless $NODE_INFO{$lnn}{'Cores'}; # Assign the number of cores for this node my $core_count = $NODE_INFO{$lnn}{'Cores'}; # Turn the performance loads into percents based on the number of cores for the node my $highest = sprintf("%d", ($stats{$lnn}{'highest'} / $core_count) * 100); my $average = sprintf("%d", ($stats{$lnn}{'average'} / $core_count) * 100); # Set a flag if the average is above threshold if ($average > $THRESHOLD) { $average_above_threshold{$lnn} = 1; } # Increment cluster totals $total_of_averages += $average; $total_of_highest += $highest; $node_count++; # Verbose output verbose("Highest: ${highest}%"); verbose("Average: ${average}%"); } # Warn if any flag is set for an average being above threshold if (%average_above_threshold) { warning("The following nodes have an average load of over ${THRESHOLD}%: ". compressRange(keys %average_above_threshold)); } # Assign average performance across cluster my ($average_of_averages, $average_of_highest); $average_of_averages = int($total_of_averages / $node_count) if defined $total_of_averages and defined $node_count; $average_of_highest = int($total_of_highest / $node_count) if defined $total_of_highest and defined $node_count; # Verbose output verboseHeader($CLUSTER_INFO{'Name'}); verbose("Average of highest across the cluster: ${average_of_highest}%") if defined $average_of_highest; verbose("Average of averages across the cluster: ${average_of_averages}%") if defined $average_of_averages; # Output Check result printResults(); } # End checkPerformance $CHECKS{'checkStoragePools'}{'Description'} = "Checks storage pools for health/capacity/unprovisioned drive issues"; sub checkStoragePools { return if wasCalled(); printCheck("Storage Pools"); gatherIsiStatPools(); checkKB501267(); my @fail_unhealthy_pools = (); my @warn_unhealthy_pools = (); my @fail_hdd_used_pools = (); my @warn_hdd_used_pools = (); my @fail_ssd_used_pools = (); my @warn_ssd_used_pools = (); my @fail_used_pools = (); my @warn_used_pools = (); foreach my $name (keys %{$CLUSTER_INFO{'Pools'}}) { my %pool = %{$CLUSTER_INFO{'Pools'}{$name}}; if ($pool{'Health'} and $pool{'Health'} =~ /S|D|R|U/) { push(@fail_unhealthy_pools, $name); } if ($pool{'Health'} and $pool{'Health'} =~ /M/) { push(@warn_unhealthy_pools, $name); } if ($pool{'Percent HDD Used'} and $pool{'Percent HDD Used'} >= 95) { push(@fail_hdd_used_pools, $name); } elsif ($pool{'Percent HDD Used'} and $pool{'Percent HDD Used'} >= 90) { push(@warn_hdd_used_pools, $name); } if ($pool{'Percent SSD Used'} and $pool{'Percent SSD Used'} >= 95) { push(@fail_ssd_used_pools, $name); } elsif ($pool{'Percent SSD Used'} and $pool{'Percent SSD Used'} >= 90) { push(@warn_ssd_used_pools, $name); } if (!$pool{'Percent HDD Used'} and $pool{'Percent Used'} and $pool{'Percent Used'} >= 95) { push(@fail_used_pools, $name); } elsif (!$pool{'Percent HDD Used'} and $pool{'Percent Used'} and $pool{'Percent Used'} >= 90) { push(@warn_used_pools, $name); } } if ($CLUSTER_INFO{'Unprovisioned drives'}) { warning("There are unprovisioned drives: $CLUSTER_INFO{'Unprovisioned drives'}"); } if (@fail_unhealthy_pools) { fail("The following pools are unhealthy: ".join(", ", @fail_unhealthy_pools)); } if (@warn_unhealthy_pools) { warning("The following pools are missing drives: ".join(", ", @warn_unhealthy_pools)); } if (@fail_hdd_used_pools) { critical("The following pools have HDD usage over 95%: ".join(", ", @fail_hdd_used_pools)); } if (@warn_hdd_used_pools) { critical("The following pools have HDD usage over 90%: ".join(", ", @warn_hdd_used_pools)); } if (@fail_ssd_used_pools) { critical("The following pools have SSD usage over 95%: ".join(", ", @fail_ssd_used_pools)); } if (@warn_ssd_used_pools) { critical("The following pools have SSD usage over 90%: ".join(", ", @warn_ssd_used_pools)); } if (@fail_used_pools) { critical("The following pools have usage over 95%: ".join(", ", @fail_used_pools)); } if (@warn_used_pools) { critical("The following pools have usage over 90%: ".join(", ", @warn_used_pools)); } # Output Check result printResults(); } # End checkStoragePools ### # checkProcesses: # Checks whether a node owns the OpenSM master role. One node in the cluster must own the OpenSM master role. # https://support.emc.com/kb/16664 # # Checks for MCP Running on all nodes # https://support.emc.com/kb/175249 - OneFS: isi_mcp fails to start on one or more nodes in the cluster $CHECKS{'checkProcesses'}{'Description'} = "Checks for OpenSM master, MCP, and isi_upgrade_d process related issues"; sub checkProcesses { return if wasCalled(); printCheck("Processes"); gatherNetworkInterfaces(); # Gather information my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'ps auxww | grep -E "isi_mcp|opensm|isi_upgrade_d" | grep -v grep']); } else { %gather = gatherCluster(qq[2>&1 grep -H -E "isi_mcp|opensm|isi_upgrade_d" */ps]); } # Analyze information for issues. my $opensm_master = 0; my @nodes_with_upgrade_process = (); my @nodes_with_not_enough_isi_mcp = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } my $mcp_processes = 0; foreach my $line (@{$gather{$lnn}}) { verbose($line); # Check for two isi_mcp processes on each node: $mcp_processes++ if ($line =~ /isi_mcp/); # Check for an OpenSM Master if ($line =~ /opensm:\s+(0x\S+)\s+master/) { $opensm_master++; } # Check for isi_upgrade_d processes, there should be none running push(@nodes_with_upgrade_process, $lnn) if ($line =~ /isi_upgrade_d/); } # Check that we found a master and child isi_mcp process push(@nodes_with_not_enough_isi_mcp, $lnn) if ($mcp_processes < 2); } warning("isi_upgrade_d process running on the following nodes:".join(",", @nodes_with_upgrade_process)) if (@nodes_with_upgrade_process); fail("isi_mcp process count less than 2 on the following nodes:".join(",", @nodes_with_not_enough_isi_mcp)) if (@nodes_with_not_enough_isi_mcp); # Check that opensm master processes matches the number of IB Interfaces on each node if ($CLUSTER_INFO{'Active IB Interfaces'} and $opensm_master != $CLUSTER_INFO{'Active IB Interfaces'}) { fail("Number of OpenSM Masters does not match the number of InfiniBand Interfaces in use"); } info("Refer to KB210509 (https://support.emc.com/kb/210509) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkProcesses sub checkRFC2307 { return if wasCalled(); verboseHeader("RFC2307 Settings"); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi_gconfig|grep ActiveDirectory"); } elsif (-e "local/isi_gconfig") { @gather = gatherLocal("2>&1 grep ActiveDirectory local/isi_gconfig"); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { @gather = gatherLocal("2>&1 grep ActiveDirectory local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } # Analyze information for issues. my ($sfu_support_global, $allocate_gids_global, $allocate_uids_global, $store_sfu_mappings_global); my %ADS = (); foreach my $line (@gather) { $allocate_gids_global = $1 if ($line =~ /ActiveDirectory\.AllocateGids .*= (\S+)$/); $allocate_uids_global = $1 if ($line =~ /ActiveDirectory\.AllocateUids .*= (\S+)$/); $sfu_support_global = $1 if ($line =~ /ActiveDirectory\.SFUSupport .*= (\S+)$/); $store_sfu_mappings_global = $1 if ($line =~ /ActiveDirectory\.StoreSFUMappings .*= (\S+)$/); my $id = $1 if ($line =~ /ActiveDirectory\.Instance\._key\.([^\.]+)\./); next unless $id; $ADS{$id}{'Name'} = $1 if ($line =~ /$id\._name .*= (\S+)$/); $ADS{$id}{'AllocateGids'} = $1 if ($line =~ /$id\._dword\.AllocateGids\.value .*= (\S+)$/); $ADS{$id}{'AllocateUids'} = $1 if ($line =~ /$id\._dword\.AllocateUids\.value .*= (\S+)$/); $ADS{$id}{'SFUSupport'} = $1 if ($line =~ /$id\._sz\.SFUSupport\.value .*= (\S+)$/); $ADS{$id}{'StoreSFUMappings'} = $1 if ($line =~ /$id\._dword\.StoreSFUMappings\.value .*= (\S+)$/); } # Setup global/defaults if they were not gathered $allocate_gids_global = "1" unless defined $allocate_gids_global; $allocate_uids_global = "1" unless defined $allocate_uids_global; $sfu_support_global = "none" unless defined $sfu_support_global; $store_sfu_mappings_global = "0" unless defined $store_sfu_mappings_global; # Update ADS values if they were not gathered to the global/defaults and then see if it meets criteria to be impacted by mapping issues my @ads_with_issues = (); verbose(sprintf("%-20s %15s %15s %15s %16s", "Provider Name", "AllocateGids", "AllocateUids", "SFUSupport", "StoreSFUMappings")); verbose(sprintf("%-20s %15s %15s %15s %16s", "Global Setttings", $allocate_gids_global, $allocate_uids_global, $sfu_support_global, $store_sfu_mappings_global)); foreach my $id (keys %ADS) { $ADS{$id}{'Name'} = $id unless defined $ADS{$id}{'Name'}; $ADS{$id}{'AllocateGids'} = $allocate_gids_global unless defined $ADS{$id}{'AllocateGids'}; $ADS{$id}{'AllocateUids'} = $allocate_uids_global unless defined $ADS{$id}{'AllocateUids'}; $ADS{$id}{'SFUSupport'} = $sfu_support_global unless defined $ADS{$id}{'SFUSupport'}; $ADS{$id}{'StoreSFUMappings'} = $store_sfu_mappings_global unless defined $ADS{$id}{'StoreSFUMappings'}; verbose(sprintf("%-20s %15s %15s %15s %16s", $ADS{$id}{'Name'}, $ADS{$id}{'AllocateGids'}, $ADS{$id}{'AllocateUids'}, $ADS{$id}{'SFUSupport'}, $ADS{$id}{'StoreSFUMappings'})); next unless $ADS{$id}{'SFUSupport'} =~ /rfc2307/i; next unless $ADS{$id}{'AllocateGids'} =~ /1/ or $ADS{$id}{'AllocateUids'} =~ /1/; push(@ads_with_issues, $ADS{$id}{'Name'}); } warning("The following ADS Authentication Providers are showing as configured for RFC2307 and are also enabled to auto-assign UIDs/GIDs: ".join(", ", @ads_with_issues)) if @ads_with_issues; addCustomerInformation("It was detected that an ADS Authentication Provider is configured for RFC2307 support and Auto-assign UIDs/GIDs are also enabled. This may lead to mapping issues which may impact client authentication to the cluster. See KB335338 (https://support.emc.com/kb/335338) for more information on how to configure OneFS and Active Directory for RFC2307 compliance.") if @ads_with_issues; } # End checkRFC2307 $CHECKS{'checkRoutingTables'}{'Description'} = "Displays routing table for each node"; sub checkRoutingTables { return if wasCalled(); printCheck("Routing Tables"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array netstat -rn]); } else { %gather = gatherCluster(q[for i in */netstat; do grep -H . $i | awk '/Routing tables/, /Active/'; done]); } # Analyze information for issues. foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { last if $line =~ /Active Internet connections/; verbose("$line"); } } # Output Check result printResults(); } # End checkRoutingTables $CHECKS{'checkServices'}{'Description'} = "Checks common services to make sure they are not disabled"; sub checkServices { # Return if was called already return if wasCalled(); # Display check name printCheck("Cluster Services"); # Gather needed information gatherServices(); # Return if services information was not populated. if (!$CLUSTER_INFO{'Services'}) { warning("No services information."); printResults(); return; } # Output details about the status of the cluster services verboseHeader("Cluster Services"); verbose(sprintf("%-30s %s", "Name", "Status")); verbose(sprintf("%-30s %s", "-"x30, "-"x10)); foreach my $service (sort keys %{$CLUSTER_INFO{'Services'}}) { verbose(sprintf("%-30s %s", $service, $CLUSTER_INFO{'Services'}{$service})); } # Check for specific services to be enabled if ($CLUSTER_INFO{'Services'}{'auth'} and $CLUSTER_INFO{'Services'}{'auth'} !~ /enabled/i) { fail("Authentication (auth) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'celog'} and $CLUSTER_INFO{'Services'}{'celog'} !~ /enabled/i) { fail("Cluster Event Log (celog) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'celog_coalescer'} and $CLUSTER_INFO{'Services'}{'celog_coalescer'} !~ /enabled/i) { fail("Cluster Event Log Coalescer (celog_coalescer) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'celog_monitor'} and $CLUSTER_INFO{'Services'}{'celog_monitor'} !~ /enabled/i) { fail("Cluster Event Log System Monitor (celog_monitor) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'celog_notification'} and $CLUSTER_INFO{'Services'}{'celog_notification'} !~ /enabled/i) { fail("Cluster Event Log Notification Daemon (celog_notification) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'ifsd'} and $CLUSTER_INFO{'Services'}{'ifsd'} !~ /enabled/i) { fail("Filesystem Daemon (ifsd) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_drive_d'} and $CLUSTER_INFO{'Services'}{'isi_drive_d'} !~ /enabled/i) { fail("Drive Daemon (isi_drive_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_drive_repurpose_d'} and $CLUSTER_INFO{'Services'}{'isi_drive_repurpose_d'} !~ /enabled/i) { fail("Drive Repurposing Daemon (isi_drive_repurpose_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_gconfig_d'} and $CLUSTER_INFO{'Services'}{'isi_gconfig_d'} !~ /enabled/i) { fail("Platform API Daemon (isi_gconfig_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_group_change_d'} and $CLUSTER_INFO{'Services'}{'isi_group_change_d'} !~ /enabled/i) { fail("Group Change Daemon (isi_group_change_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_job_d'} and $CLUSTER_INFO{'Services'}{'isi_job_d'} !~ /enabled/i) { fail("Job Daemon (isi_job_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_mcp'} and $CLUSTER_INFO{'Services'}{'isi_mcp'} !~ /enabled/i) { fail("MCP Failsafe Daemon (isi_mcp) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_protohealth_d'} and $CLUSTER_INFO{'Services'}{'isi_protohealth_d'} !~ /enabled/i) { fail("Protocol Availability (isi_protohealth_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_rpc_d'} and $CLUSTER_INFO{'Services'}{'isi_rpc_d'} !~ /enabled/i) { fail("RPS Daemon (isi_rpc_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_tardis_d'} and $CLUSTER_INFO{'Services'}{'isi_tardis_d'} !~ /enabled/i) { warning("Tardis Configuration (isi_tardis_d) service is not enabled."); } if ($CLUSTER_INFO{'Services'}{'isi_upgrade_d'} and $CLUSTER_INFO{'Services'}{'isi_upgrade_d'} !~ /disabled/i) { warning("Upgrade Daemon (isi_upgrade_d) service is enabled."); } if ($CLUSTER_INFO{'Services'}{'lwsm'} and $CLUSTER_INFO{'Services'}{'lwsm'} !~ /enabled/i) { fail("Likewise Service Manager (lwsm) service is not enabled."); } # Issue where post upgrade to OneFS 8, isi_cbind_d service goes missing, KB 488106. if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") == 0 and !$CLUSTER_INFO{'Services'}{'isi_cbind_d'}) { fail("isi_cbind_d service is missing in isi services -a, manually verify and, if missing, escalate and reference KB 488106"); } # Issue in OneFS 8 when Cloudpools is licensed but cloud pools services are disabled if ((($CLUSTER_INFO{'Services'}{'isi_cpool_d'} and $CLUSTER_INFO{'Services'}{'isi_cpool_d'} !~ /enabled/i) or ($CLUSTER_INFO{'Services'}{'isi_cpool_io_d'} and $CLUSTER_INFO{'Services'}{'isi_cpool_io_d'} !~ /enabled/i)) and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0 and hasLicense("CloudPools")) { fail("CloudPools is licensed but the isi_cpool_d and/or isi_cpool_io_d service is not enabled and may cause Data Unavailability per KB 496993."); } # Display the check results printResults(); } # End checkServices ### # checkSmartConnect: Check that the SC Service IPs are all assigned. $CHECKS{'checkSmartConnect'}{'Description'} = "Check that the SC Service IPs are all assigned."; sub checkSmartConnect { return if wasCalled(); gatherNetworkSubnets(); gatherNetworkPools(); gatherNetworkInterfaces(); return unless $CLUSTER_INFO{'Network Subnets'}; printCheck("SmartConnect Service IP"); debug("checkSmartConnect: Looking for each SmartConnect Service IP Address in every node."); SCIP: foreach my $subnet (keys %{$CLUSTER_INFO{'Network Subnets'}}) { debug("checkSmartConnect: Investigating Subnet $subnet"); next unless $CLUSTER_INFO{'Network Subnets'}{$subnet}{'SCIP'} and $CLUSTER_INFO{'Network Pools'}; debug("checkSmartConnect: SCIP of $CLUSTER_INFO{'Network Subnets'}{$subnet}{'SCIP'}"); next if $CLUSTER_INFO{'Network Subnets'}{$subnet}{'SCIP'} =~ /0\.0\.0\.0/; my $members = 0; foreach my $pool (keys %{$CLUSTER_INFO{'Network Pools'}}) { next unless $CLUSTER_INFO{'Network Pools'}{$pool}{'Subnet'} and $CLUSTER_INFO{'Network Pools'}{$pool}{'Ifaces'} and $CLUSTER_INFO{'Network Pools'}{$pool}{'Subnet'} eq $subnet; debug("checkSmartConnect: Network Pool $pool found under Subnet $subnet"); $members += scalar(@{$CLUSTER_INFO{'Network Pools'}{$pool}{'Ifaces'}}); } debug("checkSmartConnect: Subnet $subnet has $members members"); next SCIP unless $members; debug("checkSmartConnect: Searching for SCIP: $CLUSTER_INFO{'Network Subnets'}{$subnet}{'SCIP'}"); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$lnn}{'Network'}; foreach my $interface (keys %{$NODE_INFO{$lnn}{'Network'}}) { foreach my $address (@{$NODE_INFO{$lnn}{'Network'}{$interface}{'addresses'}}) { if ($address eq $CLUSTER_INFO{'Network Subnets'}{$subnet}{'SCIP'}) { debug("checkSmartConnect: Found SCIP $address on Node $lnn Interface $interface"); next SCIP; } } } } # Couldn't locate a SC Service IP Address fail("The SmartConnect Service IP Address, $CLUSTER_INFO{'Network Subnets'}{$subnet}{'SCIP'}, could not be located on any node"); } info("Refer to KB210518 (https://support.emc.com/kb/210518) for details.") if (@WARN or @FAIL); # Output Check result printResults(); } # End checkSmartConnect ### # checkSnapshot: Checks some basic snapshot details $CHECKS{'checkSnapshot'}{'Description'} = "Checks if the Snapshot count is approaching cluster limit of 20,000, whether Autodelete is set to yes, and checks snapshot logs. Checks snapshot logs for EIN/EIO/EDEADLK/Failed to create snapshot"; sub checkSnapshot { return if wasCalled(); printCheck("Snapshot"); gatherSnapshotCount(); gatherSnapshotSettings(); gatherSnapshotUsage(); checkSnapshotLog(); if ($CLUSTER_INFO{'Snapshots'}{'Count'} and $CLUSTER_INFO{'Snapshots'}{'Count'} >= 20000) { critical("Total snapshot count is $CLUSTER_INFO{'Snapshots'}{'Count'}, more than the limit for snapshots per cluster of 20,000."); } elsif ($CLUSTER_INFO{'Snapshots'}{'Count'} and $CLUSTER_INFO{'Snapshots'}{'Count'} >= 19000) { critical("Total snapshot count is $CLUSTER_INFO{'Snapshots'}{'Count'}, approaching the limit for snapshots per cluster of 20,000."); } elsif ($CLUSTER_INFO{'Snapshots'}{'Count'} and $CLUSTER_INFO{'Snapshots'}{'Count'} >= 15000) { warning("Total snapshot count is $CLUSTER_INFO{'Snapshots'}{'Count'}, approaching the limit for snapshots per cluster of 20,000."); } if ($CLUSTER_INFO{'Snapshots'}{'Autodelete'} and $CLUSTER_INFO{'Snapshots'}{'Autodelete'} !~ /Yes/) { warning("Snapshot Autodelete not set to Yes"); } # Output Check result printResults(); } # End checkSnapshot sub checkSnapshotLog { return if wasCalled(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array "awk ']. qq[/EINVAL/ {EINVAL++; EINVALMR=\\\$1}]. qq[/EIO/ {EIO++; EIOMR=\\\$1}]. qq[/EDEADLK/ {EDEADLK++; EDEADLKMR=\\\$1}]. qq[/Failed to create snapshot/ {FAIL++; FAILMR=\\\$1}]. qq[END {]. qq[print \\\"EINVAL:\\\" EINVAL;]. qq[print \\\"EINVALMR:\\\" EINVALMR;]. qq[print \\\"EIO:\\\" EIO;]. qq[print \\\"EIOMR:\\\" EIOMR;]. qq[print \\\"EDEADLK:\\\" EDEADLK;]. qq[print \\\"EDEADLKMR:\\\" EDEADLKMR;]. qq[print \\\"FAIL:\\\" FAIL;]. qq[print \\\"FAILMR:\\\" FAILMR;]. qq[}]. qq[' /var/log/isi_snapshot_d.log"]); } else { %gather = gatherCluster(qq[for i in */varlog.tar/log/isi_snapshot_d.log;do ]. qq[echo -n \$i; awk '/EINVAL/ {EINVAL++; EINVALMR=\$1} END {print ":EINVAL:" EINVAL " EINVALMR:" EINVALMR}' \$i;]. qq[echo -n \$i; awk '/EIO/ {EIO++; EIOMR=\$1} END {print ":EIO:" EIO " EIOMR:" EIOMR}' \$i;]. qq[echo -n \$i; awk '/EDEADLK/ {EDEADLK++; EDEADLKMR=\$1} END {print ":EDEADLK:" EDEADLK " EDEADLKMR:" EDEADLKMR}' \$i;]. qq[echo -n \$i; awk '/Failed to create snapshot/ {FAIL++; FAILMR=\$1} END {print ":FAIL:" FAIL " FAILMR:" FAILMR}' \$i;]. qq[done]); } # Analyze information for issues. my %errors = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $gather{$lnn}; foreach my $line (@{$gather{$lnn}}) { $errors{$lnn}{'EINVAL'}{'Count'} = $1 if ($line =~ /EINVAL:(\d+)/); $errors{$lnn}{'EINVAL'}{'MR'} = $1 if ($line =~ /EINVALMR:(\S+)/); $errors{$lnn}{'EIO'}{'Count'} = $1 if ($line =~ /EIO:(\d+)/); $errors{$lnn}{'EIO'}{'MR'} = $1 if ($line =~ /EIOMR:(\S+)/); $errors{$lnn}{'EDEADLK'}{'Count'} = $1 if ($line =~ /EDEADLK:(\d+)/); $errors{$lnn}{'EDEADLK'}{'MR'} = $1 if ($line =~ /EDEADLKMR:(\S+)/); $errors{$lnn}{'FAIL'}{'Count'} = $1 if ($line =~ /FAIL:(\d+)/); $errors{$lnn}{'FAIL'}{'MR'} = $1 if ($line =~ /FAILMR:(\S+)/); } } foreach my $lnn (keys %errors) { foreach my $type (keys %{$errors{$lnn}}) { warning("Node $lnn: Found $errors{$lnn}{$type}{'Count'} $type errors, the most recent was $errors{$lnn}{$type}{'MR'}"); } } } # End checkSnapshotLog $CHECKS{'checkSPNs'}{'Description'} = "Displays a list of SPNs, only useful when run with --extra"; sub checkSPNs { return if wasCalled(); printCheck("SPN List"); if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") < 0) { verbose("This check has not been validated for OneFS versions prior to OneFS 7.1."); printResults(); return; } # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { @gather = gatherLocal(qq[for i in `${SUDO} isi auth ads list -a -z|awk '{print \$1}'`; do ${SUDO} isi auth ads spn list --domain=\$i; done 2>&1]); } else { @gather = gatherLocal(qq[for i in `${SUDO} isi auth ads list -a -z|awk '{print \$1}'`; do ${SUDO} isi auth ads spn list \$i; done 2>&1]); } } else { @gather = gatherLocal(qq[2>&1 awk '/auth ads spn list/, /ads trusts list/' local/isi_auth]); } # Analyze information for issues. verboseHeader("SPN List"); foreach my $line (@gather) { next if $line =~ /auth ads trusts list/; verbose($line); } # Output Check result printResults(); } # End checkSPNs $CHECKS{'checkSRS'}{'Description'} = "Checks if the cluster is SRS enabled or not. Displays DialHome information used by RPS; whether ConnectEMC and SupportIQ are currently enabled or disabled."; sub checkSRS { return if wasCalled(); printCheck("DialHome & Remote Connectivity"); gatherServices(); gatherRemoteSupport(); gatherSRS(); gatherEventChannel(); details("Current Service States:"); details(" SupportIQ is $CLUSTER_INFO{'Services'}{'supportiq'}") if $CLUSTER_INFO{'Services'}{'supportiq'}; details(" ConnectEMC Service is $CLUSTER_INFO{'Services'}{'connectemc'}") if $CLUSTER_INFO{'Services'}{'connectemc'}; details(" SRS Service Daemon is $CLUSTER_INFO{'Services'}{'isi_esrs_d'}") if $CLUSTER_INFO{'Services'}{'isi_esrs_d'}; # SRSv3 (isi esrs) is enabled if ($CLUSTER_INFO{'SRS'}{'Enabled'}) { details(" SRS (isi esrs) is enabled"); # Flag if the isi_esrs_d service is not enabled if ($CLUSTER_INFO{'Services'}{'isi_esrs_d'} =~ /disabled/i) { critical("SRS (isi esrs) is enabled, but the SRS Daemon service is disabled and should be enabled for DialHome functionality"); critical(" To enable the SRS Daemon service run the following command: isi services -a isi_esrs_d enable"); } # Flag if there is no subnet/pool configured if (!$CLUSTER_INFO{'SRS'}{'Pools'}) { critical("SRS (isi esrs) is enabled, but no Gateway Access Pools are configured"); } } # Remote Support (isi remotesupport) is enabled if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'}) { details(" RemoteSupport (isi remotesupport) is enabled"); # Flag if the connectemc service is disabled if ($CLUSTER_INFO{'Services'}{'connectemc'} =~ /disabled/i) { critical("RemoteSupport (isi remotesupport) is enabled, but the ConnectEMC service is disabled and should be enabled for DialHome functionality"); critical(" To enable the ConnectEMC service run the following command: isi services -a connectemc enable"); } # Flag if there is no subnet/pool configured if (!$CLUSTER_INFO{'RemoteSupport'}{'Subnet'} and !$CLUSTER_INFO{'RemoteSupport'}{'Gateway Access Pools'}) { critical("RemoteSupport (isi remotesupport) is enabled, but no Remote Support Subnet or Gateway Access Pools are configured"); } # Flag an issue where the RemoteSupport event channel is not enabled and # the cluster is configured for remotesupport if ($CLUSTER_INFO{'Event Channels'}) { my $enabled = 0; foreach my $id (sort keys %{$CLUSTER_INFO{'Event Channels'}}) { if ($CLUSTER_INFO{'Event Channels'}{$id}{'Name'} eq "RemoteSupport" and $CLUSTER_INFO{'Event Channels'}{$id}{'Enabled'} eq "Yes" and $CLUSTER_INFO{'Event Channels'}{$id}{'Type'} eq "connectemc") { $enabled = 1; } } if (!$enabled) { critical("RemoteSupport (isi remotesupport) is enabled, but the RemoteSupport event channel is disabled and should be enabled for DialHome functionality."); critical(" To enable the RemoteSupport event channel run the following command: isi event channel modify RemoteSupport --enabled=true"); } } # Flag if the cluster is running OneFS 8.1+ and should be using SRS not RemoteSupport if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") >= 0 and !$CLUSTER_INFO{'SRS'}{'Enabled'}) { warning("RemoteSupport (isi remotesupport) is enabled, and the cluster is running a version of OneFS 8.1 or later."); warning(" In OneFS 8.1 and later, SRS (isi esrs) is the recommended method for Remote Support functionality."); warning(" Review the 'Remote support' section of the administration guides for OneFS $CLUSTER_INFO{'OneFS'} for details on how to enable this functionality."); } } if (!($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} or $CLUSTER_INFO{'SRS'}{'Enabled'})) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") < 0) { details(" RemoteSupport/SRS is not supported prior to OneFS 7.1"); } else { details(" RemoteSupport/SRS is not enabled"); } } if ($CLUSTER_INFO{'Services'}{'supportiq'} and $CLUSTER_INFO{'Services'}{'supportiq'} =~ /enabled/i and !($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.0") >= 0)) { info("SupportIQ is enabled but reached EOSL on June 30, 2016"); info("Only one remote connectivity system should be enabled, however SupportIQ and Remote Support are both enabled.") if $CLUSTER_INFO{'RemoteSupport'}{'Enabled'}; } # Verbose output for isi esrs if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") >= 0) { verboseHeader("SRSv3/SRS (isi esrs)"); verbose("Enabled: ". ($CLUSTER_INFO{'SRS'}{'Enabled'} or "No")); verbose("Primary SRS Gateway: ". ($CLUSTER_INFO{'SRS'}{'Primary Gateway'} or "-")); verbose("Secondary SRS Gateway: ". ($CLUSTER_INFO{'SRS'}{'Secondary Gateway'} or "-")); verbose("Gateway Access Pools: ". ($CLUSTER_INFO{'SRS'}{'Pools'} or "-")); } # Verbose output for isi remotesupport if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") >= 0) { verboseHeader("SRSv2/RemoteSupport (isi remotesupport):"); verbose("Enabled: ". ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} or "No")); verbose("Primary SRS Gateway: ". ($CLUSTER_INFO{'RemoteSupport'}{'Primary Gateway'} or "-")); verbose("Secondary SRS Gateway: ". ($CLUSTER_INFO{'RemoteSupport'}{'Secondary Gateway'} or "-")); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { verbose("Gateway Access Pools: ". ($CLUSTER_INFO{'RemoteSupport'}{'Gateway Access Pools'} or "-")); } else { verbose("Remote Support Subnet: ". ($CLUSTER_INFO{'RemoteSupport'}{'Subnet'} or "-")); } } # Verbose output for event channels if ($CLUSTER_INFO{'Event Channels'}) { verboseHeader("Event Channels"); my $format = "%-5s %-10s %-10s %s"; verbose(sprintf($format, "ID", "Type", "Enabled", "Name")); verbose(sprintf($format, "-"x5, "-"x10, "-"x10, "-"x20)); foreach my $id (sort keys %{$CLUSTER_INFO{'Event Channels'}}) { verbose(sprintf($format, $CLUSTER_INFO{'Event Channels'}{$id}{"ID"}, $CLUSTER_INFO{'Event Channels'}{$id}{"Type"}, $CLUSTER_INFO{'Event Channels'}{$id}{"Enabled"}, $CLUSTER_INFO{'Event Channels'}{$id}{"Name"})); } } # Output Check result printResults(); } # End checkSRS $CHECKS{'checkSyncIQ'}{'Description'} = "Gathers source and target SyncIQ information and reports partners of SyncIQ. Checks for an issue where having a large number of SyncIQ report files causes a tar process to delay other upgrade processes potentially leaving the cluster in a temporary DU scenario for an extended period of time."; sub checkSyncIQ { return if wasCalled(); checkKB521778(); printCheck("SyncIQ"); gatherSyncIQ(); gatherNetworkPools(); checkKB495911(); # Gather incoming SyncIQ details my @source_clusters = (); verboseHeader("Incoming SyncIQ"); if ($CLUSTER_INFO{'SyncIQ'}{'Target'}) { verbose(sprintf("%-32s %-10s %-20s %s", "Policy ID", "Timestamp", "Policy Name", "Source Cluster")); foreach my $pid (keys %{$CLUSTER_INFO{'SyncIQ'}{'Target'}}) { my %siq_incoming = %{$CLUSTER_INFO{'SyncIQ'}{'Target'}{$pid}}; next unless $siq_incoming{'Source Cluster'} and $siq_incoming{'Timestamp'} and $siq_incoming{'Policy Name'}; verbose(sprintf("%-32s %-10s %-20s %s", $pid, $siq_incoming{'Timestamp'}, $siq_incoming{'Policy Name'}, $siq_incoming{'Source Cluster'})); next unless $RPS{'Timestamp'} - $siq_incoming{'Timestamp'} < 7776000; my @results = grep(/$siq_incoming{'Source Cluster'}/i, @source_clusters); next if @results; push(@source_clusters, $siq_incoming{'Source Cluster'}); } } details("Target for: ".join(", ", @source_clusters)) if @source_clusters; # Gather a list of SyncIQ clusters that are a target from this cluster my @target_clusters = (); # And store the policy name if the policy last job state was not successful my @failed_policies = (); # And store the policy name if the policy restricts use to a Dynamic network pool my @restricted_to_dynamic_pool = (); verboseHeader("Outgoing SyncIQ"); verbose(sprintf("%-32s %-10s %-20s %s", "Policy ID", "Timestamp", "Policy Name", "Destination Cluster")); if ($CLUSTER_INFO{'SyncIQ'}{'Source'}) { foreach my $pid (keys %{$CLUSTER_INFO{'SyncIQ'}{'Source'}}) { my %siq_outgoing = %{$CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}}; # Next if there is no policy name next unless $siq_outgoing{'Name'} and $siq_outgoing{'Timestamp'} and $siq_outgoing{'Target'}; verbose(sprintf("%-32s %-10s %-20s %s", $pid, $siq_outgoing{'Timestamp'}, $siq_outgoing{'Name'}, $siq_outgoing{'Target'})); # Next if the most recent start was over 90 days ago next unless $RPS{'Timestamp'} - $siq_outgoing{'Timestamp'} < 7776000; # Gather a list of policies that are not finished/scheduled/running/- push(@failed_policies, $siq_outgoing{'Name'}) if ($siq_outgoing{'Enabled'} and $siq_outgoing{'Last Job State'} and $siq_outgoing{'Last Job State'} !~ /^(-|finished|scheduled|running)/i); # Gather a list of policies that are restricted to use Dynamic IP pools push(@restricted_to_dynamic_pool, $siq_outgoing{'Name'}) if ($siq_outgoing{'Enabled'} and $siq_outgoing{'Restrict Source Nodes'} and $CLUSTER_INFO{'Network Pools'}{$siq_outgoing{'Restrict Source Nodes'}}{'Allocation'} and $CLUSTER_INFO{'Network Pools'}{$siq_outgoing{'Restrict Source Nodes'}}{'Allocation'} =~ /dynamic/i); # Compile a list of target clusters my @results = grep(/$siq_outgoing{'Target'}/i, @target_clusters); next if @results; push(@target_clusters, $siq_outgoing{'Target'}) } } details("Source to: ".join(", ", @target_clusters)) if @target_clusters; # Flag for this cluster being part of SyncIQ, used in the upgrade plan $RPS{'SyncIQ'} = "Yes" if @source_clusters or @target_clusters; # Check if there are only Dynamic pools configured when it is determined there is SyncIQ in use my $only_dynamic_pools_available = 0; if (@target_clusters or @source_clusters) { $only_dynamic_pools_available = 1; foreach (keys %{$CLUSTER_INFO{'Network Pools'}}) { next unless $CLUSTER_INFO{'Network Pools'}{$_}{'Allocation'}; if ($CLUSTER_INFO{'Network Pools'}{$_}{'Allocation'} =~ /static/i) { $only_dynamic_pools_available = 0; } } } if ($only_dynamic_pools_available or @restricted_to_dynamic_pool) { # Flag a known issue with 7.2.1.0 - 7.2.1.2. if ((!$RPS{'Target'} and $CLUSTER_INFO{'OneFS'} =~ /^7\.2\.1\.[012]$/) or ($RPS{'Target'} and $RPS{'Target'} =~ /^7\.2\.1\.[012]$/)) { fail("There is a known issue with SyncIQ and Dynamic pools in OneFS 7.2.1.0-7.2.1.2, see KB480064 for details."); } # Warn about not using static pools for replication my $msg = "SyncIQ best practice is to use Static network pools"; if ($only_dynamic_pools_available) { $msg .= ", and all network pools are Dynamic."; } else { $msg .= ", and the following SyncIQ policies are restricted to use a network pool set to Dynamic: ".join(", ", @restricted_to_dynamic_pool); } warning($msg); addCustomerInformation($msg); } if (@INFO and $RPS{'Target'} and compareVersions($CLUSTER_INFO{'OneFS'}, "8") < 0) { warning("Pause or cancel any SyncIQ jobs that are running on your EMC Isilon cluster before upgrading the OneFS operating system."); addCustomerInformation("Pause or cancel any SyncIQ jobs that are running on your EMC Isilon cluster before upgrading the OneFS operating system."); } info("Refer to KB489059 (https://support.emc.com/kb/489059) for details.") if (@WARN or @FAIL); printResults(); } # End checkSyncIQ ### # checkTimeDrift: Determines if the times on nodes are out of sync $CHECKS{'checkTimeDrift'}{'Description'} = "Checks time drift between nodes"; sub checkTimeDrift { return if wasCalled(); printCheck("Cluster Time Drift"); gatherUptime(); # Compare times between nodes, maintain a lowest and highest value my $min_time; my $max_time; foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$lnn}{'time'}; my $time = 0; $time += 12 * 60 if $NODE_INFO{$lnn}{'time'} =~ /PM/; $time += 12 * $1 + $2 if $NODE_INFO{$lnn}{'time'} =~ /^(\d+):(\d+)/; if (not defined $min_time) { $min_time = $max_time = $time; } if ($time < $min_time) { $min_time = $time; } if ($time > $max_time) { $max_time = $time; } } # Flag issues with time drift my $time_drift; my $results = "Unknown"; if (not defined $min_time or not defined $max_time) { fail("Unable to determine time drift"); } else { $time_drift = $max_time - $min_time; $results = "$time_drift Minutes"; if ($time_drift >= 5) { fail("Time drift between nodes is greater than 5 minutes"); } elsif ($time_drift >= 3) { warning("Time drift between nodes is between 3 and 5 minutes"); } } # Output Check result printResults($time_drift); } # End checkTimeDrift ### # checkTimeSync: Checks if the cluster is enabled to sync to an external server $CHECKS{'checkTimeSync'}{'Description'} = "Checks if the cluster is enabled to sync to an external server"; sub checkTimeSync { return if wasCalled(); printCheck("Cluster Time Sync"); gatherSMBTime(); gatherNTPConfig(); gatherNTPdXML(); # Warn if no external time sync is found if (!$CLUSTER_INFO{'NTP Servers'} and !$CLUSTER_INFO{'SMBTime'}) { warning("This cluster is not configured to synchronize time externally."); } # Output Check result printResults(); } # End checkTimeSync ### # checkTimeZone: Checks for missing time zone in target OneFS code level. $CHECKS{'checkTimeZone'}{'Description'} = "Checks for missing time zone in target OneFS code level."; sub checkTimeZone { return if wasCalled(); printCheck("Cluster Time Zone"); gatherArrayXML(); # No target OneFS version, not impacted. if (!$RPS{'Target'}) { debug("checkTimeZone: No target OneFS version, not impacted."); } # Target OneFS version not a version known as impacted elsif ($RPS{'Target'} !~ /^8\.0\.0\.[012345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/) { debug("checkTimeZone: Target OneFS version not a version known as impacted"); } # Time zone was not gathered elsif (!$CLUSTER_INFO{'Timezone'}) { fail("Unable to determine time zone, manually verify"); } # Time zones missing in OneFS 8.0.0.1, OneFS 8.0.0.2, OneFS 8.0.0.3, OneFS 8.0.0.4, OneFS 8.0.0.5, OneFS 8.0.1.0, OneFS 8.0.1.1, and OneFS 8.1.0.0 elsif ($RPS{'Target'} =~ /^8\.0\.0\.[12345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/) { my @Timezone = ("Brazil/East","Brazil/DeNoronha","Brazil/West","Brazil/Acre","Pacific/Samoa","PRC","Mexico/General","Mexico/BajaSur","Mexico/BajaNorte","Singapore","Canada/Yukon","Canada/Pacific","Canada/Central","Canada/Eastern","Canada/Mountain","Canada/Atlantic","Canada/Newfoundland","Canada/East-Saskatchewan","Canada/Saskatchewan","America/Santa_Isabel","America/Ensenada","Chile/Continental","Chile/EasterIsland","GMT0","US/Arizona","US/Pacific","US/Central","US/Alaska","US/Eastern","US/Samoa","US/East-Indiana","US/Mountain","US/Aleutian","US/Indiana-Starke","US/Michigan","US/Hawaii","Antarctica/South_Pole","Asia/Rangoon"); if (grep {$_ eq $CLUSTER_INFO{'Timezone'}} @Timezone) { fail("$CLUSTER_INFO{'Timezone'} timezone found on this cluster, an upgrade to OneFS 8.0.0.[012345], 8.0.1.[01], or 8.1.0.0 should not be performed."); } } # Time zones missing in OneFS 8.0.0.0 elsif ($RPS{'Target'} =~ /^8\.0\.0\.0$/) { my @Timezone = ("Europe/Tiraspol","Europe/Belfast","Brazil/East","Brazil/DeNoronha","Brazil/West","Brazil/Acre","W-SU","GB-Eire","Pacific/Ponape","Pacific/Samoa","Pacific/Truk","Pacific/Yap","Israel","Portugal","ROC","Universal","PRC","Kwajalein","Iceland","ROK","Mexico/General","Mexico/BajaSur","Mexico/BajaNorte","Egypt","Navajo","Poland","Jamaica","Singapore","Africa/Timbuktu","Africa/Asmera","Australia/Canberra","Australia/South","Australia/NSW","Australia/North","Australia/Tasmania","Australia/West","Australia/Yancowinna","Australia/Victoria","Australia/ACT","Australia/LHI","Australia/Queensland","Libya","Canada/Yukon","Canada/Pacific","Canada/Central","Canada/Eastern","Canada/Mountain","Canada/Atlantic","Canada/Newfoundland","Canada/East-Saskatchewan","Canada/Saskatchewan","NZ","Japan","Zulu","Turkey","GMT+0","America/Jujuy","America/Buenos_Aires","America/Montreal","America/Rosario","America/Virgin","America/Cordoba","America/Mendoza","America/Knox_IN","America/Indianapolis","America/Shiprock","America/Catamarca","America/Argentina/ComodRivadavia","America/Atka","America/Coral_Harbour","America/Fort_Wayne","America/Ensenada","America/Porto_Acre","America/Louisville","Chile/Continental","Chile/EasterIsland","GMT0","UCT","Cuba","Hongkong","Atlantic/Faeroe","Atlantic/Jan_Mayen","GMT","Eire","US/Arizona","US/Pacific","US/Central","US/Alaska","US/Eastern","US/Samoa","US/East-Indiana","US/Mountain","US/Aleutian","US/Indiana-Starke","US/Michigan","US/Hawaii","Iran","Greenwich","NZ-CHAT","Antarctica/South_Pole","Asia/Katmandu","Asia/Kashgar","Asia/Calcutta","Asia/Thimbu","Asia/Ashkhabad","Asia/Dacca","Asia/Saigon","Asia/Tel_Aviv","Asia/Chungking","Asia/Ulan_Bator","Asia/Ujung_Pandang","Asia/Harbin","Asia/Macao","Asia/Chongqing","GMT-0","GB"); if (grep {$_ eq $CLUSTER_INFO{'Timezone'}} @Timezone) { fail("$CLUSTER_INFO{'Timezone'} timezone found on this cluster, an upgrade to OneFS 8.0.0.0 should not be performed."); } } # Output Check result info("Refer to KB502177 (https://support.emc.com/kb/502177) for details.") if (@WARN or @FAIL); printResults($CLUSTER_INFO{'Timezone'} or "Unknown"); } # End checkTimeZone ### # checkUpgrade: Checks for in progress upgrade related issues. # Warns if the isi_upgrade_d service is enabled. # Fails if not in a committed state. # Fails if an upgrade activity is already in progress. # Calls checkJobStatus to validate running Upgrade job. # Calls checkProcesses to validate running isi_upgrade_d processes. $CHECKS{'checkUpgrade'}{'Description'} = "Checks for in progress upgrade related issues. Warns if the isi_upgrade_d service is enabled. Fails if not in a committed state. Fails if an upgrade activity is already in progress.Checks the fs_fmt_version, an odd or zero fs_fmt_version is problematic"; sub checkUpgrade { return if wasCalled(); checkProcesses(); checkJobStatus(); checkUpgradeAgentPort(); printCheck("Upgrade Status"); gatherServices(); gatherUpgradeCluster(); verboseHeader("checkUpgrade"); verbose("isi_upgrade_d is $CLUSTER_INFO{'Services'}{'isi_upgrade_d'}") if $CLUSTER_INFO{'Services'}{'isi_upgrade_d'}; if ($CLUSTER_INFO{'Services'} and $CLUSTER_INFO{'Services'}{'isi_upgrade_d'} and $CLUSTER_INFO{'Services'}{'isi_upgrade_d'} eq "Enabled") { warning("Service isi_upgrade_d is currently Enabled."); } verbose("Cluster Upgrade State: $CLUSTER_INFO{'Cluster Upgrade State'}") if $CLUSTER_INFO{'Cluster Upgrade State'}; if ($CLUSTER_INFO{'Cluster Upgrade State'} and $CLUSTER_INFO{'Cluster Upgrade State'} ne "committed") { fail("Cluster Upgrade State is currently $CLUSTER_INFO{'Cluster Upgrade State'}."); } verbose("Current Upgrade Activity: $CLUSTER_INFO{'Current Upgrade Activity'}") if $CLUSTER_INFO{'Current Upgrade Activity'}; if ($CLUSTER_INFO{'Current Upgrade Activity'} and !($CLUSTER_INFO{'Current Upgrade Activity'} eq "-" or $CLUSTER_INFO{'Current Upgrade Activity'} =~ /No activity/i)) { fail("Current Upgrade Activity is currently $CLUSTER_INFO{'Current Upgrade Activity'}"); } checkFileSystemFormatVersion(); if (@WARN or @FAIL) { info("Refer to KB210510 (https://support.emc.com/kb/210510) for details."); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { info("See also the Isilon Customer Troubleshooting Guide for upgrades from OneFS 6.5, 7.0, 7.1, or 7.2: https://support.emc.com/docu60948"); } else { info("See also the Isilon Customer Troubleshooting Guide for upgrades from OneFS 8 and later: https://support.emc.com/docu83028"); } } # Output Check result printResults(); } # End checkUpgrade $CHECKS{'checkUpgradeAgentPort'}{'Description'} = "Checks the port used by the isi_upgrade_agent_d daemon to ensure it is not in use by other processes"; sub checkUpgradeAgentPort { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0; printCheck("Upgrade Agent Port"); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[ 2>&1 ${SUDO} isi_for_array ]. qq[ sockstat -P udp -p 15100 ]); } else { %gather = gatherCluster(qq[ 2>&1 grep -H -E 'USER|15100' */sockstat ]); } # Port 15100 should be used for isi_upgrade_agent_d. my $port = 15100; my %nodes_with_problematic_pids = (); # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { # Header line if ($line =~ /^\s*USER/) { verbose($line); next; } # Split the line into fields my @fields = split(/\s+/, $line); # Skip if there are less than 7 fields next if scalar(@fields) < 7; # Skip unless the PROTOCOL field is udp next unless $fields[4] =~ /udp/; # Skip unless the LOCAL ADDRESS field ends with the port next unless $fields[5] =~ /:$port$/; # Add this line to the verbose output verbose($line); # Skip if the COMMAND is isi_upgrad (short for isi_upgrade_agent_d) next if $fields[1] =~ /isi_upgrad/; # Beyond this point there must be a process other than # isi_upgrade_agent_d that is on the port. # Gather for each node the problematic PID and COMMAND $nodes_with_problematic_pids{$lnn}{$fields[2]} = $fields[1]; } } # Flag issues identified fail("A process other than the isi_upgrade_agent_d is on port $port on ". "nodes: ".compressRange(keys %nodes_with_problematic_pids)) if %nodes_with_problematic_pids; foreach my $lnn (keys %nodes_with_problematic_pids) { my @pidlist = (); foreach my $pid (keys %{$nodes_with_problematic_pids{$lnn}}) { push(@pidlist, "$nodes_with_problematic_pids{$lnn}{$pid} ($pid)"); } fail("Node $lnn: ".join(", ", @pidlist)); } # Output Check result info("Refer to KB523984 (https://support.emc.com/kb/523984) for details.") if (@WARN or @FAIL); printResults(); } # End checkUpgradeAgentPort $CHECKS{'checkUptime'}{'Description'} = "Checks node uptime, warning over 200 days, flags uptime ETAs"; sub checkUptime { return if wasCalled(); printCheck("Node Uptime"); gatherUptime(); gatherETA202452(); gatherETA209918(); gatherETA491747(); gatherDTA526050(); # ETA 209918 if ($RPS{'ETA'}{'209918'} and $RPS{'ETA'}{'209918'}{'Message'}) { info(@{$RPS{'ETA'}{'209918'}{'Message'}}); } # ETA 202452 if ($RPS{'ETA'}{'202452'} and $RPS{'ETA'}{'202452'}{'Message'}) { info(@{$RPS{'ETA'}{'202452'}{'Message'}}); } # ETA 491747 if ($RPS{'ETA'}{'491747'} and $RPS{'ETA'}{'491747'}{'Message'}) { info(@{$RPS{'ETA'}{'491747'}{'Message'}}); } # DTA 526050 if ($RPS{'ETA'}{'526050'} and $RPS{'ETA'}{'526050'}{'Message'}) { info(@{$RPS{'ETA'}{'526050'}{'Message'}}); } my $highest_uptime = 0; foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); verbose("Gathered uptime: $NODE_INFO{$lnn}{'uptime_gathered'}") if $NODE_INFO{$lnn}{'uptime_gathered'}; next unless $NODE_INFO{$lnn}{'uptime'}; my $uptime_seconds = $NODE_INFO{$lnn}{'uptime'}; my @calculated_uptime = (); if ($uptime_seconds > 86400) { my $days = int($uptime_seconds / 86400); $uptime_seconds -= $days * 86400; push(@calculated_uptime, "$days days") } if ($uptime_seconds > 3600) { my $hours = int($uptime_seconds / 3600); $uptime_seconds -= $hours * 3600; push(@calculated_uptime, "$hours hours") } if ($uptime_seconds > 60) { my $minutes = int($uptime_seconds / 60); $uptime_seconds -= $minutes * 60; push(@calculated_uptime, "$minutes minutes") } if ($uptime_seconds > 0) { push(@calculated_uptime, "$uptime_seconds seconds") } verbose("Calculated uptime: ".join(", ", @calculated_uptime)) if @calculated_uptime; $highest_uptime = $NODE_INFO{$lnn}{'uptime'} if ($NODE_INFO{$lnn}{'uptime'} > $highest_uptime); } # Not used, but may come in handy debug("checkUptime: Predicted date for 248.5 day uptime bug: ".scalar(localtime($RPS{'Runtime'} + 21384000 - $highest_uptime))); # Warn about/provide information for node uptime and ETAs if ($highest_uptime > 42940800) { if ($RPS{'ETA'}{'202452'}) { warning("ETA202452: Nodes over 497 days uptime."); push(@{$RPS{'Proactive Reboot'}{'Priority'}}, "During any activity that requires node reboots it is possible that all remaining nodes in the cluster restart simultaneously, without warning, due to ETA 202452."); } if ($RPS{'ETA'}{'526050'}) { warning("DTA526050: Nodes over 497 days uptime."); } } elsif ($highest_uptime > 41731200) { if ($RPS{'ETA'}{'202452'}) { warning("ETA202452: Nodes approaching 497 days uptime."); } if ($RPS{'ETA'}{'526050'}) { warning("DTA526050: Nodes approaching 497 days uptime."); } } if ($highest_uptime > 21470400 and $RPS{'ETA'}{'209918'}) { warning("ETA209918: Nodes over 248.5 days uptime."); push(@{$RPS{'Proactive Reboot'}{'Priority'}}, "During any activity that requires node reboots it is possible that all remaining nodes in the cluster restart simultaneously, without warning, due to ETA 209918."); } elsif ($highest_uptime > 20217600 and $RPS{'ETA'}{'209918'}) { warning("ETA209918: Nodes approaching 248.5 days uptime."); } if ($highest_uptime > 21470400 and $RPS{'ETA'}{'491747'}) { warning("ETA491747: Nodes over 248.5 days uptime."); } elsif ($highest_uptime > 20217600 and $RPS{'ETA'}{'491747'}) { warning("ETA491747: Nodes approaching 248.5 days uptime."); } # Proactive reboot only for versions of OneFS prior to 8.0 if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0 and $highest_uptime > 17280000) { push(@{$RPS{'Proactive Reboot'}{'Priority'}}, "A proactive reboot preceding all other activities is strongly recommended due to the high uptime of nodes."); warning("Highest node uptime over 200 days.") if !@WARN; @{$RPS{'Proactive Reboot'}{'Nodes'}} = keys %NODE_INFO; } info("Refer to KB210571 (https://support.emc.com/kb/210571) for details") if (@WARN or @FAIL); my $highest_uptime_days = int($highest_uptime / 86400); printResults("$highest_uptime_days days"); } # End checkUptime ### # gatherArrayXML: Gathers information from /etc/ifs/array.xml sub gatherArrayXML { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 grep -h -E 'name|encoding|timezone|interface|netmask|low|high|mtu|device|array_|guid|serial|compliance' /etc/ifs/array.xml]); } else { @gather = gatherLocal(qq[2>&1 grep -h -E 'name|encoding|timezone|interface|netmask|low|high|mtu|device|array_|guid|serial|compliance' */etc.tar/etc/ifs/array.xml */etcifs.tar/ifs/array.xml]); } # Analyze information for issues. my $name = ""; my $cluster_name = ""; my $device = 0; my $lnn = 0; my $serial = ""; my $devid = 0; foreach my $line (@gather) { # Gather the cluster name if ($line =~ /([^<]+)Name=$1"); } # Gather Internal Network Interface details elsif ($line =~ /Interfaces->${name}->Flags=$1"); } } elsif ($name and $line =~ /(.*?)<\/netmask>/) { $CLUSTER_INFO{'Interfaces'}{$name}{'Netmask'} = $1; debug("CLUSTER_INFO->Interfaces->${name}->Netmask=$1"); } elsif ($name and $line =~ /(.*?)<\/low>/) { $CLUSTER_INFO{'Interfaces'}{$name}{'Low'} = $1; debug("CLUSTER_INFO->Interfaces->${name}->Low=$1"); } elsif ($name and $line =~ /(.*?)<\/high>/) { $CLUSTER_INFO{'Interfaces'}{$name}{'High'} = $1; debug("CLUSTER_INFO->Interfaces->${name}->High=$1"); } elsif ($name and $line =~ /(.*?)<\/mtu>/) { $CLUSTER_INFO{'Interfaces'}{$name}{'MTU'} = $1; debug("CLUSTER_INFO->Interfaces->${name}->MTU=$1"); } elsif ($line =~ /<\/interface/) { $name = ""; } # Gather Node/Device details elsif ($line =~ //) { $device = 1; } elsif ($line =~ /<\/device>/) { if ($lnn) { $NODE_INFO{$lnn}{'Name'} = "$cluster_name-$lnn"; debug("NODE_INFO->${lnn}->Name=$NODE_INFO{$lnn}{'Name'}"); } if ($lnn and $devid) { $NODE_INFO{$lnn}{'ID'} = $devid; debug("NODE_INFO->${lnn}->ID=$NODE_INFO{$lnn}{'ID'}"); $LNN_FROM_DEVID{$devid} = $lnn; } if ($lnn and $serial) { $NODE_INFO{$lnn}{'Serial'} = $serial; debug("NODE_INFO->${lnn}->Serial=$NODE_INFO{$lnn}{'Serial'}"); } $device = 0; $devid = 0; $serial = ""; $lnn = 0; } elsif ($device and $line =~ /([^<]+)([^<]+)([^<]+)(.*)<\/encoding>/) { $CLUSTER_INFO{'Encoding'} = $1; debug("CLUSTER_INFO->Encoding=$1"); } # Gather Cluster Timezone elsif ($line =~ /(.*)<\/timezone>/) { $CLUSTER_INFO{'Timezone'} = $1; debug("CLUSTER_INFO->Timezone=$1"); } # Gather cluster GUID elsif (!$device and $line =~ /([^<]+)GUID=$1"); } # Gather compliance mode elsif ($line =~ /(.*)<\/compliance>/) { $CLUSTER_INFO{'Compliance'} = $1; debug("CLUSTER_INFO->Compliance=$1"); } } # Populate the number of nodes based on what was gathered $CLUSTER_INFO{'Node Count'} = scalar(keys %NODE_INFO); debug("CLUSTER_INFO->Node Count=$CLUSTER_INFO{'Node Count'}"); @LNNS = sort {$a <=> $b} keys %NODE_INFO; debug("Sorted lnn list: ".join(",",@LNNS)); } # End gatherArrayXML sub gatherATime { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} sysctl efs.bam | grep atime"); } else { @gather = gatherLocal("2>&1 grep -H atime local/ifsvar_etc.tar/etc/mcp/override/sysctl.conf"); } # Analyze information for issues. foreach my $line (@gather) { if ($line =~ /atime_enabled.*?(\d+)$/) { $CLUSTER_INFO{'ATime'}{'Enabled'} = $1; debug("CLUSTER_INFO->ATime->Enabled=$1"); } if ($line =~ /atime_grace_period.*?(\d+)$/) { $CLUSTER_INFO{'ATime'}{'Grace Period'} = $1; debug("CLUSTER_INFO->ATime->Grace Period=$1"); } } } # End gatherATime ### # gatherComplianceMode: Gathers information for cluster being compliance mode sub gatherComplianceMode { return if wasCalled(); my $gather = ""; if ($RPS{'Live'}) { chomp($gather = `sysctl isi.compliance 2>/dev/null | awk '{print \$2}'`); } else { chomp($gather = `cat local/isi.compliance 2>/dev/null | awk '{print \$2}'`); } $CLUSTER_INFO{'Compliance'} = $gather; debug("CLUSTER_INFO->Compliance=$CLUSTER_INFO{'Compliance'}"); } # End gatherComplianceMode sub gatherEventChannel { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0; my $match ="ID|Name|Type|Enabled"; # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} ]. qq[isi event channel list -v ]. qq[| grep -E "$match" ]); } else { @gather = gatherLocal(qq[2>&1 ]. qq[grep -E "$match" ]. qq[local/isi_event_channel ]); } # Analyze information for issues. my $id = ""; foreach my $line (@gather) { # Skip unless the line is a match next unless $line =~ /^\s*($match)/; # Remove any leading whitespaces $line =~ s/^\s*//; # Split into a name/value pair my ($name, $value) = split(/:\s*/, $line, 2); # Assign the ID $id = $value if $name eq "ID"; # Skip if no ID is found next unless $id; # Assign values $CLUSTER_INFO{'Event Channels'}{$id}{$name} = $value; debug("CLUSTER_INFO->Event Channels->${id}->${name}=$value"); } } # End gatherEventChannel ### # gatherEventGroups8: Gathers events on OneFS 8+ clusters. sub gatherEventGroups8 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { debug("gatherEventGroups8: Skipping. OneFS version prior to 8."); return; } # Gather event group information my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi event groups list| grep critical | grep '\\-\\-'"); } else { @gather = gatherLocal("2>&1 grep -h critical */isi_event_groups | grep '\\-\\-'"); } foreach my $line (@gather) { next if ($line =~ /^ID/ or $line =~ /^-/ or $line =~ /^Total/); next unless ($line =~ /^(\d+)\s+([\d\/]+\s+[\d:]+)\s+(--)\s+(\S+)\s+(\d+)\s+(\d+)\s+(critical)$/); my ($id, $started, $ended, $causes, $lnn, $events, $severity) = ($1, $2, $3, $4, $5, $6, $7); # Assign out variables $CLUSTER_INFO{'Events'}{$id}{'Started'} = $started; debug("CLUSTER_INFO->Events->${id}->Started=$started"); $CLUSTER_INFO{'Events'}{$id}{'Ended'} = $ended; debug("CLUSTER_INFO->Events->${id}->Ended=$ended"); $CLUSTER_INFO{'Events'}{$id}{'Severity'} = $severity; debug("CLUSTER_INFO->Events->${id}->Severity=$severity"); } } # End gatherEventGroups8 ### # gatherEvents: Gathers events sub gatherEvents { return if wasCalled(); gatherEvents7(); gatherEvents8(); } # End gatherEvents ### # gatherEvents7: Gathers events on OneFS 7 and earlier clusters. sub gatherEvents7 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { debug("gatherEvents7: Skipping for OneFS versions 8 and later."); return; } # Gather event group information my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} isi events list -w | grep '\\-\\- .* C ']); } else { @gather = gatherLocal(qq[2>&1 grep -h '\\-\\- .* C ' */isi_alerts */isi_events | sort | uniq]); } # Iterate through gather foreach my $line (@gather) { next unless ($line =~ /^([\d\.]+)\s+([\d\/]+\s+[\d:]+)\s+(--)\s+(C)\s+(\S+)\s+(.*?)$/); my ($id, $started, $ended, $severity, $lnn, $message) = ($1, $2, $3, $4, $5, $6); # Skip event if already recorded next if $CLUSTER_INFO{'Events'}{$id}; # Assign out variables $CLUSTER_INFO{'Events'}{$id}{'Started'} = $started; debug("CLUSTER_INFO->Events->${id}->Started=$started"); $CLUSTER_INFO{'Events'}{$id}{'Ended'} = $ended; debug("CLUSTER_INFO->Events->${id}->Ended=$ended"); $CLUSTER_INFO{'Events'}{$id}{'Severity'} = $severity; debug("CLUSTER_INFO->Events->${id}->Severity=$severity"); push(@{$CLUSTER_INFO{'Events'}{$id}{'Lnn'}}, $lnn); debug("CLUSTER_INFO->Events->${id}->Lnn=$lnn"); push(@{$CLUSTER_INFO{'Events'}{$id}{'Message'}}, $message); debug("CLUSTER_INFO->Events->${id}->Message=$message"); } } # End gatherEvents7 ### # gatherEvents8: Gathers events on OneFS 8+ clusters. sub gatherEvents8 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { debug("gatherEvents8: Skipping. OneFS version prior to 8."); return; } gatherEventGroups8(); return unless $CLUSTER_INFO{'Events'}; # Gather event information my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi event events list --severity=critical|grep '".join("\\|", keys %{$CLUSTER_INFO{'Events'}})."'") if $CLUSTER_INFO{'Events'}; } else { @gather = gatherLocal("2>&1 sqlite3 local/celog_ifsvar_db.tar/ifs/_ifsvar/db/celog/events.db 'select devid, coalesce_instanceid, message from events where coalesce_instanceid in (".join(",", keys %{$CLUSTER_INFO{'Events'}}).")'") if $CLUSTER_INFO{'Events'}; } # Iterate through gather foreach my $line (@gather) { next if ($line =~ /^ID/ or $line =~ /^Total/ or $line =~ /^-/); my ($lnn, $eventgroup, $message) = ""; if ($line =~ /^(\d+)\|(\d+)\|(.*)$/) { ($lnn, $eventgroup, $message) = ($1, $2, $3); $lnn = $LNN_FROM_DEVID{$1} if $LNN_FROM_DEVID{$1}; } elsif ($line =~ /^\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+(\S+)\s+(.*)$/) { ($lnn, $eventgroup, $message) = ($1, $2, $3); } $lnn = "All" if defined $lnn and $lnn eq "0"; next unless $lnn and $eventgroup and $message; # Assign out variables my @results = grep {$_ eq $lnn or $_ eq "All"} @{$CLUSTER_INFO{'Events'}{$eventgroup}{'Lnn'}}; unless (@results) { push(@{$CLUSTER_INFO{'Events'}{$eventgroup}{'Lnn'}}, $lnn); debug("CLUSTER_INFO->Events->${eventgroup}->Lnn=$lnn"); } @results = grep {$_ eq $message} @{$CLUSTER_INFO{'Events'}{$eventgroup}{'Message'}}; unless (@results) { push(@{$CLUSTER_INFO{'Events'}{$eventgroup}{'Message'}}, $message); debug("CLUSTER_INFO->Events->${eventgroup}->Message=$message"); } } } # End gatherEvents8 sub gatherFlexnetConfig { # Gather information from cluster or from logset my @gather = (); my $grep_string = "sbr-enabled|tcp-port|rebalance-delay|groupnet|subnet|name|server-side-dns-search|dns-cache-enabled|gateway|description|sc-service|sc-zone|auth-zone|mtu|alloc-method|aggregation-mode|sc-failover-policy|sc-failback-mode|sc-auto-unsuspend-delay|low|high"; if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} cat /ifs/.ifsvar/modules/flexnet/flx_config.xml | grep -E '$grep_string']); } else { @gather = gatherLocal(qq[2>&1 cat local/ifsvar_modules.tar/modules/flexnet/flx_config.xml */etcifs.tar/ifs/flx_config.xml */etc.tar/etc/ifs/flx_config.xml | grep -E '$grep_string']); } # Analyze information for issues. my @level = (); my @names = (); my %tmp = (); foreach my $line (@gather) { last if ($line =~ /<\/flexnet/); if ($line =~ /<(groupnet|subnet|network-pool)>/) { push(@level, $1); } elsif ($line =~ /<\/(groupnet|subnet|network-pool)>/) { if ($level[$#level] ne $1) { print "Something went wrong in gatherFlexnetConfig town, send help."; } %{$CLUSTER_INFO{'FlexNet'}{$level[$#level]}{join(".",@names)}} = %{$tmp{$level[$#level]}}; %{$tmp{$level[$#level]}} = (); pop(@level); pop(@names); } elsif ($line =~ /(\d)FlexNet->SBR=$1"); } elsif ($line =~ /(\d+)FlexNet->Rebalance Delay=$1"); } elsif ($line =~ /(\d+)FlexNet->TCP Ports=$1"); } elsif ($line =~ /(\d+)FlexNet->Server Side DNS Search=$1"); } } elsif ($line =~ /(\d+)FlexNet->DNS Cache Enabled=$1"); } } next unless scalar(@level); if ($line =~ /([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)&1 ${SUDO} isi stat -d -v]); } else { @gather = gatherLocal(qq[2>&1 ${SUDO} isi --timeout=300 stat -p -v]); } } else { @gather = gatherLocal(qq[2>&1 cat */isi_stat-d */isi_stat-p]); } debug(@gather); # Analyze information for issues. my $node_group_name = ""; my $seen_cluster_name = 0; foreach my $line (@gather) { # Gather cluster name if ($line =~ /^Cluster Name:\s+(\S+)$/) { last if $seen_cluster_name; $CLUSTER_INFO{'Name'} = $1; $seen_cluster_name = 1; debug("CLUSTER_INFO->Name=$CLUSTER_INFO{'Name'}"); } # Gather cluster health if ($line =~ /^Cluster Health:\s+\[\s*(\S+)\s*\]$/) { if (!$CLUSTER_INFO{'Health'}) { $CLUSTER_INFO{'Health'} = $1; debug("CLUSTER_INFO->Healthy=$CLUSTER_INFO{'Health'}"); } } # Start gathering pool information if ($line =~ /^(?:Node|Disk) (?:Group|Pool) Name:\s+(\S+)(?:\s+Protection:\s+(\S+))?/) { $node_group_name = $1; if (defined $2) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'Protection'} = $2; debug("CLUSTER_INFO->Pools->${node_group_name}->Protection=$2"); } } # Only proceed from this point if we have gathered a group name next unless $node_group_name; # Gather VHS details for the group if ($line =~ /^VHS Size:\s+(\S+)/) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'VHS'} = $1; debug("CLUSTER_INFO->Pools->${node_group_name}->VHS=$1"); } # Gather %Used details for the group if ($line =~ /^Used:.*?(\d+)%(?:.*?(\d+)%)?/) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'Percent Used'} = $1; debug("CLUSTER_INFO->Pools->${node_group_name}->Percent Used=$1"); if (defined $2) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'Percent HDD Used'} = $1; debug("CLUSTER_INFO->Pools->${node_group_name}->Percent HDD Used=$1"); $CLUSTER_INFO{'Pools'}{$node_group_name}{'Percent SSD Used'} = $2; debug("CLUSTER_INFO->Pools->${node_group_name}->Percent SSD Used=$2"); } } next if ($line =~ /^Name/); if ($line =~ /^\S+\s*\|/) { my @columns = split(/\s*\|\s*/, $line); $CLUSTER_INFO{'Pools'}{$node_group_name}{'Health'} = $columns[1]; debug("CLUSTER_INFO->Pools->${node_group_name}->Health=$columns[1]"); $CLUSTER_INFO{'Pools'}{$node_group_name}{'HDD Storage'} = $columns[5]; debug("CLUSTER_INFO->Pools->${node_group_name}->HDD Storage=$columns[5]"); $CLUSTER_INFO{'Pools'}{$node_group_name}{'SSD Storage'} = $columns[$#columns]; debug("CLUSTER_INFO->Pools->${node_group_name}->SSD Storage=$columns[$#columns]"); } next unless ($line =~ /^\s*(\d+)\|/); my $lnn = $1; push(@{$CLUSTER_INFO{'Pools'}{$node_group_name}{'Members'}}, $lnn); debug("CLUSTER_INFO->Pools->${node_group_name}->Members=$lnn"); $NODE_INFO{$lnn}{'Pool'} = $node_group_name; debug("NODE_INFO->${lnn}->Pool=$node_group_name"); my @columns = split(/\s*\|\s*/, $line); $NODE_INFO{$lnn}{'Health'} = $columns[2]; debug("NODE_INFO->${lnn}->Health=$NODE_INFO{$lnn}{'Health'}"); $NODE_INFO{$lnn}{'HDD Storage'} = $columns[6]; debug("NODE_INFO->${lnn}->HDD Storage=$NODE_INFO{$lnn}{'HDD Storage'}"); $NODE_INFO{$lnn}{'SSD Storage'} = $columns[$#columns]; debug("NODE_INFO->${lnn}->SSD Storage=$NODE_INFO{$lnn}{'SSD Storage'}"); if ($NODE_INFO{$lnn}{'HDD Storage'} =~ /(\d+)%/ or $NODE_INFO{$lnn}{'SSD Storage'} =~ /(\d+)%/) { $NODE_INFO{$lnn}{'Percent Used'} = $1; debug("NODE_INFO->${lnn}->Percent Used=$1"); } if ($NODE_INFO{$lnn}{'SSD Storage'} =~ /L3/) { $NODE_INFO{$lnn}{'L3'} = 1; debug("NODE_INFO->${lnn}->L3=1"); } } } # End gatherIsiStatPools ### # gatherJobStatus: Gathers job information. sub gatherJobStatus { return if wasCalled(); gatherJobStatus7(); gatherJobStatus8(); } # End gatherJobStatus ### # gatherJobStatus7: Gathers job information in OneFS prior to 8. sub gatherJobStatus7 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { debug("gatherJobStatus7: Skipping. OneFS version 8 or later."); return; } my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "7.0") >= 0 ) { @gather = gatherLocal("2>&1 ${SUDO} isi_classic job status -v"); } else { @gather = gatherLocal("2>&1 ${SUDO} isi job status -v"); } } else { @gather = gatherLocal("2>&1 cat local/isi_job_status"); } # Analyze information for issues. my $state = ""; foreach (@gather) { $state = "Running" if /^Running jobs:/; $state = "Paused" if /^Paused and waiting jobs:/; $state = "" if /^No running jobs|^No paused or waiting jobs|^No failed jobs/; last if /^Failed jobs:/; last if /^Recent job results/; if (/^(\w+\[\d+\])/) { debug("CLUSTER_INFO->Jobs->${state}=$1"); push(@{$CLUSTER_INFO{'Jobs'}{$state}}, $1); } } } # End gatherJobStatus7 ### # gatherJobStatus8: Gathers job information in OneFS 8. sub gatherJobStatus8 { return if wasCalled(); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0) { debug("gatherJobStatus8: Skipping. OneFS version prior to 8."); return; } my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi job status -v"); } else { @gather = gatherLocal("2>&1 cat local/isi_job_status"); } my $unpack_template = ""; foreach (@gather) { $unpack_template = generateUnpackTemplate($_, "ID","Type","State","Impact","Pri","Phase","Running Time") if /^ID/; next if /^-*$/; last if /^Recent finished jobs/; last if /^No running or queued jobs./; next unless $unpack_template; next unless /^\d/; eval { my (undef, $type, $state, undef, undef, undef, undef) = unpack($unpack_template, $_); debug("CLUSTER_INFO->Jobs->${state}=$type"); push(@{$CLUSTER_INFO{'Jobs'}{$state}}, $type); }; } } # End gatherJobStatus8 ### # gatherLicense: Gather and store license details sub gatherLicense { gatherLicense7(); gatherLicense80(); gatherLicense81(); } # End gatherLicense ### # gatherLicense7: Gather and store license details sub gatherLicense7 { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") < 0; my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi license"); } else { @gather = gatherLocal("2>&1 cat local/isi_license"); } # Analyze information for issues. my $unpack_template = ""; my $min_length = 0; foreach my $line (@gather) { # Generate an unpack template from the header line if ($line =~ /^Module/) { $min_length = length($line); $unpack_template = generateUnpackTemplate($line, "Module", "License", "Configuration", "Expiration"); } # Skip header/foot lines and skip until an unpack templace is available next if $line =~ /^\s*-/; next if $line =~ /^Module/; next unless $unpack_template; # Expand the line to be at least as long as the header line to avoid unpack issues $line = sprintf("%-${min_length}s", $line); # Unpack the line my @unpacked_line = (); eval { @unpacked_line = unpack($unpack_template, $line); }; $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Status'} = $unpacked_line[1]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Status=$unpacked_line[1]"); if ($unpacked_line[3]) { $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Expiration'} = $unpacked_line[3]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Expiration=$unpacked_line[3]"); } } } # End gatherLicense7 ### # gatherLicense80: Gather and store license details sub gatherLicense80 { return if wasCalled(); return unless $CLUSTER_INFO{'OneFS'} =~ /^8\.0/; my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi license list"); } else { @gather = gatherLocal("2>&1 cat local/isi_license"); } # Analyze information for issues. my $unpack_template = ""; my $min_length = 0; foreach my $line (@gather) { # Generate an unpack template from the header line if ($line =~ /^Name/) { $min_length = length($line); $unpack_template = generateUnpackTemplate($line, "Name", "Status", "Expiration"); } # Skip header/foot lines and skip until an unpack templace is available next if $line =~ /^\s*-/; next if $line =~ /^Total/; next if $line =~ /^Name/; next unless $unpack_template; # Expand the line to be at least as long as the header line to avoid unpack issues $line = sprintf("%-${min_length}s", $line); # Unpack the line my @unpacked_line = (); eval { @unpacked_line = unpack($unpack_template, $line); }; $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Status'} = $unpacked_line[1]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Status=$unpacked_line[1]"); if ($unpacked_line[2] ne "-") { $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Expiration'} = $unpacked_line[2]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Expiration=$unpacked_line[2]"); } } } # End gatherLicense80 ### # gatherLicense81: Gather and store license details sub gatherLicense81 { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") >= 0; my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi license list"); } else { @gather = gatherLocal("2>&1 cat local/isi_license"); } # Analyze information for issues. my $unpack_template = ""; my $min_length = 0; foreach my $line (@gather) { # Generate an unpack template from the header line if ($line =~ /^Module/) { $min_length = length($line); $unpack_template = generateUnpackTemplate($line, "Module", "Licensed", "Actual", "Status", "Expiration"); } # Gather OneFS License ID if ($line =~ /OneFS License ID:\s*(\S+)$/ and $1 ne "-") { $CLUSTER_INFO{'OneFS License ID'} = $1; debug("CLUSTER_INFO->OneFS License ID=$1"); } # Gather if cluster has a "Valid Signature" if ($line =~ /Valid Signature:\s*Yes$/) { $CLUSTER_INFO{'Valid Signature'} = "Yes"; debug("CLUSTER_INFO->Valid Signature=Yes"); } # Skip header/foot lines and skip until an unpack templace is available next if $line =~ /^[\s-]*$/; next if $line =~ /^Total/; next if $line =~ /^Module/; next unless $unpack_template; # Unpack the line eval { # Expand the line to be at least as long as the header line to avoid unpack issues $line = sprintf("%-${min_length}s", $line); my @unpacked_line = (); @unpacked_line = unpack($unpack_template, $line); # Assign values out $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Status'} = $unpacked_line[3]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Status=$unpacked_line[3]"); if ($unpacked_line[4] ne "-") { $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Expiration'} = $unpacked_line[4]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Expiration=$unpacked_line[4]"); } }; } } # End gatherLicense81 ### # gatherNetworkInterfaces: Gathers network interface information. sub gatherNetworkInterfaces { return if wasCalled(); # Gather IP Addresses from ifconfig my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array ifconfig"); } else { %gather = gatherCluster("2>&1 grep -H . */ifconfig"); } # Store IP Addresses under NODE_INFO{$node}{'Network'}{$interface}{'addresses'} and # the status of each interface in NODE_INFO{$node}{'Network'}{$interface}{'status'} foreach my $node (@LNNS) { if (!$gather{$node}) { missing($node); next; } my $interface = ""; foreach my $line (@{$gather{$node}}) { $interface = $1 if ($line =~ /^\s*(\S+): flags/); next if (!$interface); # if ($line =~ /mtu (\d+)/) { # $NODE_INFO{$node}{'Network'}{$interface}{'mtu'} = $1; # debug("NODE_INFO->${node}->Network->${interface}->mtu=$1"); # } # if ($line =~ /options=(.*)/) { # $NODE_INFO{$node}{'Network'}{$interface}{'options'} = $1; # debug("NODE_INFO->${node}->Network->${interface}->options=$1"); # } if ($line =~ /lagg:\s+laggdev\s+(\S+)/) { $NODE_INFO{$node}{'Network'}{$interface}{'laggdev'} = $1; debug("NODE_INFO->${node}->Network->${interface}->laggdev=$1"); } if ($line =~ /media: (.*)/) { $NODE_INFO{$node}{'Network'}{$interface}{'media'} = $1; debug("NODE_INFO->${node}->Network->${interface}->media=$1"); } if ($line =~ /status: (.*)/) { $NODE_INFO{$node}{'Network'}{$interface}{'status'} = $1; debug("NODE_INFO->${node}->Network->${interface}->status=$1"); } if ($line =~ /inet (\S+)/) { push(@{$NODE_INFO{$node}{'Network'}{$interface}{'addresses'}}, $1); debug("NODE_INFO->${node}->Network->${interface}->addresses=$1"); } if ($line =~ /inet6 (\S+)/) { push(@{$NODE_INFO{$node}{'Network'}{$interface}{'addresses'}}, $1); debug("NODE_INFO->${node}->Network->${interface}->addresses=$1"); } if ($line =~ /^\s*laggproto\s+(\S+.*)$/) { $NODE_INFO{$node}{'Network'}{$interface}{'laggproto'} = $1; debug("NODE_INFO->${node}->Network->${interface}->laggproto=$1"); } if ($line =~ /^\s*laggport:\s+(\S.*)$/) { push(@{$NODE_INFO{$node}{'Network'}{$interface}{'laggport'}}, $1); debug("NODE_INFO->${node}->Network->${interface}->laggport=$1"); } # if ($line =~ /lladdr\s+(\S+)$/) { # $NODE_INFO{$node}{'Network'}{$interface}{'lladdr'} = $1; # debug("NODE_INFO->${node}->Network->${interface}->lladdr=$1"); # } } # Track nodes with BXE interfaces for BXE related patches push(@{$CLUSTER_INFO{'BXE Nodes'}}, $node) if ($NODE_INFO{$node}{'Network'}{'bxe0'} or $NODE_INFO{$node}{'Network'}{'bxe1'}); # Track nodes with CXGB interfaces push(@{$CLUSTER_INFO{'CXGB Nodes'}}, $node) if ($NODE_INFO{$node}{'Network'}{'cxgb0'} or $NODE_INFO{$node}{'Network'}{'cxgb1'}); # Tally how many active InfiniBand interfaces there is on each node, store this in CLUSTER_INFO{'Active IB Interfaces'} my $ibinterfaces = 0; $ibinterfaces++ if ($NODE_INFO{$node}{'Network'}{'ib0'} and $NODE_INFO{$node}{'Network'}{'ib0'}{'status'} eq 'active'); $ibinterfaces++ if ($NODE_INFO{$node}{'Network'}{'ib1'} and $NODE_INFO{$node}{'Network'}{'ib1'}{'status'} eq 'active'); if (!$CLUSTER_INFO{'Active IB Interfaces'} or $ibinterfaces > $CLUSTER_INFO{'Active IB Interfaces'}) { $CLUSTER_INFO{'Active IB Interfaces'} = $ibinterfaces; debug("CLUSTER_INFO->Active IB Interfaces=$ibinterfaces"); } } } # End gatherNetworkInterfaces ### # gatherNetworkPools: Gathers network pool information. sub gatherNetworkPools { return if wasCalled(); my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") < 0) { @gather = gatherLocal("2>&1 ${SUDO} isi networks list pool -v"); } else { @gather = gatherLocal("2>&1 ${SUDO} isi network pools list -v"); } } else { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") < 0) { @gather = gatherLocal("2>&1 cat local/isi_networks_pools"); } else { @gather = gatherLocal("2>&1 cat local/isi_network_pools"); } } my $id = ""; my $gather_pool_membership = 0; my $gather_ranges = 0; foreach (@gather) { if (/^\s*-*$/) { $id = ""; next; } if (!$id and (/^(\S+)/ or /^\s*ID: (\S+)/)) { $id = $1; debug("gatherNetworkPools: Gathering for Network Pool -> $id"); } next unless $id; if ($gather_ranges and /^\s*(\S+)/) { $gather_ranges--; push(@{$CLUSTER_INFO{'Network Pools'}{$id}{'Ranges'}}, $1); debug("CLUSTER_INFO->Network Pools->${id}->Ranges=$1"); } elsif ($gather_pool_membership and /^\s*(\S+)/) { $gather_pool_membership--; push(@{$CLUSTER_INFO{'Network Pools'}{$id}{'Ifaces'}}, $1); debug("CLUSTER_INFO->Network Pools->${id}->Ifaces=$1"); } elsif (/^\s*Groupnet: (\S+)/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Groupnet'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Groupnet=$1"); } elsif (/^\s*In Subnet: (\S+)/ or /^\s*Subnet: (\S+)/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Subnet'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Subnet=$1"); } elsif (/^\s*Allocation: (\S+)/ or /^\s*Allocation Method: (\S+)/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Allocation'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Allocation=$1"); } elsif (/^\s*Ranges: (\d+)/) { $gather_ranges = $1; debug("gatherNetworkPools: Gathering interfaces for the next $1 lines"); } elsif (/^\s*IP Ranges: (.*)$/) { @{$CLUSTER_INFO{'Network Pools'}{$id}{'Ranges'}} = split(", ", $1); debug("CLUSTER_INFO->Network Pools->${id}->Ranges=$1") foreach @{$CLUSTER_INFO{'Network Pools'}{$id}{'Ranges'}}; } elsif (/^\s*Pool Membership: (\d+)/) { $gather_pool_membership = $1; debug("gatherNetworkPools: Gathering pool membership for the next $1 lines"); } elsif (/^\s*Ifaces: (.*)$/) { @{$CLUSTER_INFO{'Network Pools'}{$id}{'Ifaces'}} = split(", ",$1); debug("CLUSTER_INFO->Network Pools->${id}->Ifaces=$1") foreach @{$CLUSTER_INFO{'Network Pools'}{$id}{'Ifaces'}}; } elsif (/^\s*Aggregation Mode: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Aggregation Mode'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Aggregation Mode=$1"); } elsif (/^\s*Access Zone: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Access Zone'} = $1; $CLUSTER_INFO{'Network Pools'}{$id}{'Access Zone'} =~ s/\s*\(.*//; debug("CLUSTER_INFO->Network Pools->${id}->Access Zone=$CLUSTER_INFO{'Network Pools'}{$id}{'Access Zone'}"); } elsif (/^\s*Suspended Nodes\s*: (.*)/ or /^\s*SC Suspended Nodes: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Suspended Nodes'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Suspended Nodes=$1"); } elsif (/^\s*Zone\s*: (.*)$/ or /^\s*SC Zone: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Zone'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Zone=$1"); } elsif (/^\s*Time to Live\s*: (.*)$/ or /^\s*SC Ttl: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Time to Live'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Time to Live=$1"); } elsif (/^\s*Service Subnet\s*: (.*)$/ or /^\s*SC Subnet: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Service Subnet'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Subnet=$1"); } elsif (/^\s*Connection Policy\s*: (.*)$/ or /^\s*SC Connect Policy: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Connection Policy'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Connection Policy=$1"); } elsif (/^\s*Failover Policy\s*: (.*)$/ or /^\s*SC Failover Policy: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Failover Policy'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Failover Policy=$1"); } elsif (/^\s*Rebalance Policy\s*: (.*)$/) { $CLUSTER_INFO{'Network Pools'}{$id}{'Rebalance Policy'} = $1; debug("CLUSTER_INFO->Network Pools->${id}->Rebalance Policy=$1"); } } } # End gatherNetworkPools ### # gatherNetworkSubnets: Gathers network subnet information. sub gatherNetworkSubnets { return if wasCalled(); my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") < 0) { @gather = gatherLocal("2>&1 ${SUDO} isi networks list subnets -v"); } else { @gather = gatherLocal("2>&1 ${SUDO} isi network subnets list -v"); } } else { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8") < 0) { @gather = gatherLocal("2>&1 cat local/isi_networks_subnets"); } else { @gather = gatherLocal("2>&1 cat local/isi_network_subnets"); } } my $id = ""; foreach (@gather) { if (/^\s*-*$/) { $id = ""; next; } if (!$id and (/^(\S+)/ or /^\s*ID: (\S+)/)) { $id = $1; debug("gatherNetworkSubnets: Gathering for Network Subnet -> $id"); } next unless $id; if (/^\s*Name: (\S+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Name'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Name=$1"); } if (/^\s*Groupnet: (\S+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Groupnet'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Groupnet=$1"); } if (/^\s*Address Family: (\S+)/ or /^\s*Addr Family: (\S+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Address Family'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Address Family=$1"); } if (/^\s*Netmask: ([0-9A-F\.:]+)/i) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Netmask'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Netmask=$1"); } if (/^\s*Subnet: ([0-9A-F\.:]+)/i or /^\s*Base Addr: ([0-9A-F\.:]+)/i) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Subnet'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Subnet=$1"); } if (/^\s*Gateway ([0-9A-F\.:]+),/i or /^\s*Gateway: ([0-9A-F\.:]+)/i) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Gateway'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Gateway=$1"); } if (/Priority (\d+)/ or /^\s*Gateway Priority: (\d+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'Priority'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->Priority=$1"); } if (/^\s*MTU: (\d+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'MTU'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->MTU=$1"); } if (/^\s*SC Service Address: ([0-9A-F\.:]+)/i or /^\s*SC Service Addr: ([0-9A-F\.:]+)/i) { $CLUSTER_INFO{'Network Subnets'}{$id}{'SCIP'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->SCIP=$1"); } if (/^\s*VLAN Tagging: (\S+)/ or /^\s*VLAN Enabled: (\S+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'VLAN Tagging'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->VLAN Tagging=$1"); } if (/^\s*VLAN ID: (\d+)/) { $CLUSTER_INFO{'Network Subnets'}{$id}{'VLAN ID'} = $1; debug("CLUSTER_INFO->Network Subnets->${id}->VLAN ID=$1"); } } } # End gatherNetworkSubnets ### # gatherNFSExport: Gathers NFS export information sub gatherNFSExport { return if wasCalled(); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi nfs exports list --verbose"); } else { @gather = gatherLocal("2>&1 cat local/isi_nfs_exports"); } my $ID = ""; foreach my $line (@gather) { $ID = $1 if ($line =~ /^\s*ID: (.*)$/ or $line =~ /^Export (.*):$/); next unless $ID; $ID = 0 if $ID =~ /default/i; if ($line =~ /^\s*Zone: (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Zone'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Zone=$1"); } elsif ($line =~ /^\s*Encoding: (.*)$/ or $line =~ /data encoding is (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Encoding=$1"); } elsif ($line =~ /^\s*Paths: (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Paths'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Paths=$1"); } elsif ($line =~ /^\s*Security Type: (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Security Type'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Security Type=$1"); } } } # End gatherNFSExport ### # gatherNTPConfig: Gathers NTP Time Server details from the ntp_config.gc file sub gatherNTPConfig { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 cat /ifs/.ifsvar/modules/ntp/ntp_config.gc /var/ifs/ntp_config.gc ]); } else { @gather = gatherLocal(qq[2>&1 cat local/ifsvar_modules.tar/modules/ntp/ntp_config.gc */gconfig_local.tar/var/ifs/ntp_config.gc]); } my @ntp_servers = (); foreach my $line (@gather) { next unless ($line =~ /^servers.*?\[\s*(.*?)\s*\]/); my $server_list = $1; $server_list =~ s/\\"//g; my @servers = split(/, /, $server_list); foreach my $server (@servers) { my @results = grep {$_ eq $server} @ntp_servers; next if @results; push(@ntp_servers, $server); } } if (@ntp_servers) { @{$CLUSTER_INFO{'NTP Servers'}} = @ntp_servers; debug("CLUSTER_INFO->NTP Servers=".join(",", @ntp_servers)); } } # End gatherNTPConfig ### # gatherNTPdXML: Gathers NTP Time Server details from the ntpd.xml file sub gatherNTPdXML { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 cat /ifs/.ifsvar/etc/mcp/override/ntpd.xml /etc/mcp/override/ntpd.xml]); } else { @gather = gatherLocal(qq[2>&1 cat local/ifsvar_etc.tar/etc/mcp/override/ntpd.xml */local_configs.tar/etc/mcp/override/ntpd.xml */override.tar/override/ntpd.xml]); } my @ntp_servers = (); foreach my $line (@gather) { next unless ($line =~ /(.*?)<\/server>$/); my $server = $1; my @results = grep {$_ eq $server} @ntp_servers; next if @results; push(@ntp_servers, $server); } if (@ntp_servers) { @{$CLUSTER_INFO{'NTP Servers'}} = @ntp_servers; debug("CLUSTER_INFO->NTP Servers=".join(",", @ntp_servers)); } } # End gatherNTPdXML ### # gatherPackageInfo: Gathers log gather information from package_info.xml sub gatherPackageInfo { return if wasCalled(); return if $RPS{'Live'}; # Gather information from the logset my @gather = gatherLocal("2>&1 cat package_info.xml"); # Analyze information for issues. foreach my $line (@gather) { $RPS{'Gather Date'} = $1 if ($line =~ /(.*)<\/gather_date>/); $RPS{'Contents'} = $1 if ($line =~ /(.*)<\/contents>/); $RPS{'Command'} = $1 if ($line =~ /(.*)<\/command>/); } # Due to a bug in some OneFS versions, package_info is not populated and Gather Date doesn't get set. if (!$RPS{'Gather Date'} and -e "package_info.xml") { $RPS{'Gather Date'} = (stat("package_info.xml"))[9]; } } # End gatherPackageInfo ### # gatherRemoteSupport: Gathers remotesupport details. sub gatherRemoteSupport { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "7.1") >= 0; # Gather information from cluster or from logset my @gather = (); if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0 and $RPS{'Live'} and -z "/ifs/.ifsvar/modules/tardis/namespaces/Gconfig.remotesupport_config.sqlite") { unlink("/ifs/.ifsvar/modules/tardis/namespaces/Gconfig.remotesupport_config.sqlite"); } if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi_gconfig -t remote-support"); } elsif (-e "local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/Gconfig.remotesupport_config.sqlite") { @gather = gatherLocal("2>&1 sqlite3 local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/Gconfig.remotesupport_config.sqlite 'select * from kv_table'"); } else { @gather = gatherLocal("2>&1 cat */gconfig_local.tar/var/ifs/remote_support.gc"); } # Analyze information for issues. foreach my $line (@gather) { if ($line =~ /^connectemc\.enabled.*true/) { $CLUSTER_INFO{'RemoteSupport'}{'Enabled'} = "Yes"; debug("CLUSTER_INFO->RemoteSupport->Enabled=Yes"); } elsif ($line =~ /^connectemc\.esrs_host.*? = (\S+)$/) { $CLUSTER_INFO{'RemoteSupport'}{'SRS Host'} = $1; debug("CLUSTER_INFO->RemoteSupport->SRS Host=$1"); } elsif ($line =~ /^connectemc\.primary_esrs_gateway.*? = (\S+)$/) { $CLUSTER_INFO{'RemoteSupport'}{'Primary Gateway'} = $1; debug("CLUSTER_INFO->RemoteSupport->Primary Gateway=$1"); } elsif ($line =~ /^connectemc\.secondary_esrs_gateway.*? = (\S+)$/) { $CLUSTER_INFO{'RemoteSupport'}{'Secondary Gateway'} = $1; debug("CLUSTER_INFO->RemoteSupport->Secondary Gateway=$1"); } elsif ($line =~ /^connectemc\.gateway_access_pools.*?\"?\[ (.*) \]\"?$/) { $CLUSTER_INFO{'RemoteSupport'}{'Gateway Access Pools'} = $1; debug("CLUSTER_INFO->RemoteSupport->Gateway Access Pools=$1"); } elsif ($line =~ /^connectemc\.remote_support_subnet.*?(\S+)$/) { $CLUSTER_INFO{'RemoteSupport'}{'Subnet'} = $1; debug("CLUSTER_INFO->RemoteSupport->Subnet=$1"); } } return unless $CLUSTER_INFO{'RemoteSupport'}; foreach my $item (keys %{$CLUSTER_INFO{'RemoteSupport'}}) { $CLUSTER_INFO{'RemoteSupport'}{$item} =~ s/\\"//g; $CLUSTER_INFO{'RemoteSupport'}{$item} =~ s/\"//g; } } # End gatherRemoteSupport ### # gatherServices: Gathers services output and stores it in $CLUSTER_INFO{'Services'} sub gatherServices { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi services -a"); } else { @gather = gatherLocal("2>&1 cat local/isi_services"); } # Analyze information for issues. foreach my $line (@gather) { next if ($line =~ /^Available Services:$/); if ($line =~ /^\s+(\S+)\s+.*(Enabled|Disabled)$/) { $CLUSTER_INFO{'Services'}{$1} = $2; debug("CLUSTER_INFO->Services->${1}=$2"); } } } # End gatherServices ### # gatherSIQPoliciesGC: Gather SyncIQ policy information sub gatherSIQPoliciesGC { return if wasCalled(); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 cat /ifs/.ifsvar/modules/tsm/config/siq-policies.gc]); } else { @gather = gatherLocal(qq[2>&1 cat local/ifsvar_modules.tar/modules/tsm/config/siq-policies.gc]); } # Gather information and store it temporarily in a siq_policies hash. my $index = ""; my %siq_policies = (); foreach (@gather) { if (/^policy\.(\d+) /) { $index = $1 } elsif (/^policy\.$index\.common\.name .*? \"(.*?)\"/) { $siq_policies{$index}{'Name'} = $1; } elsif (/^policy\.$index\.common\.pid .*? \"(.*?)\"/) { $siq_policies{$index}{'Policy ID'} = $1; } elsif (/^policy\.$index\.common\.state .*? \"(on)\"/) { $siq_policies{$index}{'Enabled'} = 1; } elsif (/^policy\.$index\.target\.dst_cluster_name .*? \"(.*?)\"/) { $siq_policies{$index}{'Target'} = $1; } elsif (/^policy\.$index\.target\.restrict_by .*? \"(.*?)\"/) { $siq_policies{$index}{'Restrict Source Nodes'} = $1 if $1 !~ /^$/; } elsif (/^policy\.$index\.scheduler\.max_reports .*? \"(.*?)\"/) { $siq_policies{$index}{'Max Count'} = $1; } elsif (/^policy\.$index\.scheduler\.rotate_report_period .*? \"(.*?)\"/) { $siq_policies{$index}{'Max Age'} = $1; } } # Assign information by Policy ID for each policy to CLUSTER_INFO foreach my $index (keys %siq_policies) { next unless $siq_policies{$index}{'Policy ID'}; my $pid = $siq_policies{$index}{'Policy ID'}; foreach my $key (keys %{$siq_policies{$index}}) { next if $key eq 'Policy ID'; $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{$key} = $siq_policies{$index}{$key}; debug("CLUSTER_INFO->SyncIQ->Source->${pid}->${key}=$siq_policies{$index}{$key}"); } } } # End gatherSIQPoliciesGC sub gatherSMBTime { return if wasCalled(); # Gather information from cluster or from logset my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array grep running /etc/mcp/override/smbtime.xml]); } else { %gather = gatherCluster(qq[2>&1 grep -H running */override.tar/override/smbtime.xml]); } # Analyze information for issues. foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $gather{$lnn}; foreach my $line (@{$gather{$lnn}}) { last if defined $CLUSTER_INFO{'SMBTime'}; if ($line =~ /(.*?)<\/running>/) { $CLUSTER_INFO{'SMBTime'} = $1; } } } } # End gatherSMBTime sub gatherSnapshotCount { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi snap list | grep '^Total'"); } else { @gather = gatherLocal("2>&1 cat local/snapshot_list | grep '^Total'"); } # Analyze information for issues. foreach my $line (@gather) { if ($line =~ /Total:\s+(\d+)$/) { $CLUSTER_INFO{'Snapshots'}{'Count'} = $1; debug("CLUSTER_INFO->Snapshots->Count=$1"); } } } # End gatherSnapshotCount sub gatherSnapshotSettings { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi snap settings view"); } else { @gather = gatherLocal("2>&1 cat local/snapshot_settings"); } # Analyze information for issues. foreach my $line (@gather) { if ($line =~ /Autodelete:\s+(\S+)$/) { $CLUSTER_INFO{'Snapshots'}{'Autodelete'} = $1; debug("CLUSTER_INFO->Snapshots->Autodelete=$1"); } } } # End gatherSnapshotSettings sub gatherSnapshotUsage { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi snap snap list -v --format table --state all | grep '^Total'"); } else { @gather = gatherLocal("2>&1 cat local/snapshot_usage | grep '^Total'"); } # Analyze information for issues. foreach my $line (@gather) { if ($line =~ /^Total:\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { $CLUSTER_INFO{'Snapshots'}{'Count'} = $1; debug("CLUSTER_INFO->Snapshots->Count=$1"); $CLUSTER_INFO{'Snapshots'}{'Size'} = $2; debug("CLUSTER_INFO->Snapshots->Size=$2"); $CLUSTER_INFO{'Snapshots'}{'Reserve'} = $3; debug("CLUSTER_INFO->Snapshots->Reserve=$3"); $CLUSTER_INFO{'Snapshots'}{'Filesystem'} = $4; debug("CLUSTER_INFO->Snapshots->Filesystem=$4"); } } } # End gatherSnapshotUsage ### # gatherSRS: Gathers SRS details sub gatherSRS { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.1") >= 0; # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig -t esrs|grep -v 'api_key']); } else { @gather = gatherLocal(qq[2>&1 cat local/isi_esrs|grep -v '^[#+-]']); } # Analyze information for issues. my %gateway_access_pools = (); foreach my $line (@gather) { if ($line =~ /^esrs\.enabled.*true/) { $CLUSTER_INFO{'SRS'}{'Enabled'} = "Yes"; debug("CLUSTER_INFO->SRS->Enabled=Yes"); } elsif ($line =~ /^esrs.primary_esrs_gateway.*? = (.*)$/) { $CLUSTER_INFO{'SRS'}{'Primary Gateway'} = $1; debug("CLUSTER_INFO->SRS->Primary Gateway=$1"); } elsif ($line =~ /^esrs.secondary_esrs_gateway.*? = (.*)$/) { $CLUSTER_INFO{'SRS'}{'Secondary Gateway'} = $1; debug("CLUSTER_INFO->SRS->Secondary Gateway=$1"); } elsif ($line =~ /^esrs\.gateway_access_pools\.(\d+)\.subnet.*? = (.*)$/) { $gateway_access_pools{$1}{'Subnet'} = $2; } elsif ($line =~ /^esrs\.gateway_access_pools\.(\d+)\.pool.*? = (.*)$/) { $gateway_access_pools{$1}{'Pool'} = $2; } } my @pools = (); foreach my $id (keys %gateway_access_pools) { next unless $gateway_access_pools{$id}{'Subnet'}; next unless $gateway_access_pools{$id}{'Pool'}; push(@pools, "$gateway_access_pools{$id}{'Subnet'}.$gateway_access_pools{$id}{'Pool'}"); } if (@pools) { $CLUSTER_INFO{'SRS'}{'Pools'} = join(", ", @pools); debug("CLUSTER_INFO->SRS->Pools=$CLUSTER_INFO{'SRS'}{'Pools'}"); } } # End gatherSRS ### # gatherStat: Gathers node information from isi stat -n/isi stat -a output. sub gatherStat { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { if (compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0) { @gather = gatherLocal("2>&1 ${SUDO} isi stat -a | grep -E '^(Member|Node|Available|Used)'"); } else { @gather = gatherLocal("2>&1 ${SUDO} isi stat -n | grep -E '^(Member|Node|Available|Used)'"); } } else { @gather = gatherLocal("2>&1 grep -E -h '^(Member|Node|Available|Used)' */isi_stat"); } # Analyze information for issues. my $lnn = 0; my $first_lnn = 0; foreach my $line (@gather) { next unless $line =~ /:/; my ($name, $value) = split(/:\s*/, $line, 2); if ($name eq "Node LNN") { next unless $value =~ /^\d+$/; $lnn = $value; last if $first_lnn == $lnn; $first_lnn = $lnn unless $first_lnn; } next unless $lnn; if ($name eq "Node ID") { $NODE_INFO{$lnn}{'ID'} = $value; debug("NODE_INFO->${lnn}->ID=$value"); } elsif ($name eq "Node Name") { $NODE_INFO{$lnn}{'Name'} = $value; debug("NODE_INFO->${lnn}->Name=$value"); } elsif ($name eq "Node Health") { $NODE_INFO{$lnn}{'Health'} = $value; debug("NODE_INFO->${lnn}->Health=$value"); } elsif ($name eq "Node SN") { $NODE_INFO{$lnn}{'Serial'} = $value; debug("NODE_INFO->${lnn}->Serial=$value"); } elsif ($name eq "Member of (Node|Disk) (Pool|Pools|Group)") { $NODE_INFO{$lnn}{'Pool'} = $value; debug("NODE_INFO->${lnn}->Pool=$value"); } elsif ($name eq "Node Capacity") { $NODE_INFO{$lnn}{'Capacity'} = $value; debug("NODE_INFO->${lnn}->Capacity=$value"); } elsif ($name eq "Available") { $NODE_INFO{$lnn}{'Available'} = $value; debug("NODE_INFO->${lnn}->Available=$value"); if ($value =~ /\D([0-9\.]+)\%/) { $NODE_INFO{$lnn}{'Percent Available'} = $1; debug("NODE_INFO->${lnn}->Percent Available=$1"); } } elsif ($name eq "Used") { $NODE_INFO{$lnn}{'Used'} = $value; debug("NODE_INFO->${lnn}->Used=$value"); if ($value =~ /\D([0-9\.]+)\%/) { $NODE_INFO{$lnn}{'Percent Used'} = $1; debug("NODE_INFO->${lnn}->Percent Used=$1"); } } } } # End gatherStat ### # gatherStatus: Gathers cluster information from isi status sub gatherStatus { return if wasCalled(); # Gather information from cluster or from logset my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi status"); } else { @gather = gatherLocal("2>&1 cat */isi_status"); } # Analyze information for issues. my $seen_cluster_name = 0; foreach my $line (@gather) { my ($hdd, $ssd, $quantifier, $numeric) = ""; if ($line =~ /^Cluster Name:\s+(\S+)$/) { last if $seen_cluster_name; $seen_cluster_name = 1; $CLUSTER_INFO{'Name'} = $1; debug("CLUSTER_INFO->Name=$CLUSTER_INFO{'Name'}"); } elsif ($line =~ /^Cluster Health:\s+\[\s*(\S+)\s*\]$/) { $CLUSTER_INFO{'Health'} = $1; debug("CLUSTER_INFO->Health=$CLUSTER_INFO{'Health'}"); } elsif ($line =~ /^VHS Size:\s+(\S+)$/) { $CLUSTER_INFO{'VHS Size'} = $1; debug("CLUSTER_INFO->VHS Size=$CLUSTER_INFO{'VHS Size'}"); } elsif ($line =~ /^Size:\s+(\S+\s+\(.*\))\s+(\S+\s+\(.*\))$/) { $CLUSTER_INFO{'HDD Storage'} = $1; debug("CLUSTER_INFO->HDD Storage=$1"); $CLUSTER_INFO{'SSD Storage'} = $2; debug("CLUSTER_INFO->SSD Storage=$2"); if ($CLUSTER_INFO{'HDD Storage'} =~ /\((\S+)\s/) { $CLUSTER_INFO{'HDD Size'} = convertToNumeric($1); debug("CLUSTER_INFO->HDD Size=$CLUSTER_INFO{'HDD Size'}"); } if ($CLUSTER_INFO{'SSD Storage'} =~ /\((\S+)\s/) { $CLUSTER_INFO{'SSD Size'} = convertToNumeric($1); debug("CLUSTER_INFO->SSD Size=$CLUSTER_INFO{'SSD Size'}"); } } elsif ($line =~ /^Used:\s+(\S+\s+\(.*\))\s+(\S+\s+\(.*\))$/) { $CLUSTER_INFO{'HDD Used'} = $1; debug("CLUSTER_INFO->HDD Used=$1"); $CLUSTER_INFO{'SSD Used'} = $2; debug("CLUSTER_INFO->SSD Used=$2"); if ($CLUSTER_INFO{'HDD Used'} =~ /\((\d+)%/) { $CLUSTER_INFO{'HDD Percent Used'} = $1; debug("CLUSTER_INFO->HDD Percent Used=$CLUSTER_INFO{'HDD Percent Used'}"); } if ($CLUSTER_INFO{'SSD Used'} =~ /\((\d+)%/) { $CLUSTER_INFO{'SSD Percent Used'} = $1; debug("CLUSTER_INFO->SSD Percent Used=$CLUSTER_INFO{'SSD Percent Used'}"); } } elsif ($line =~ /^\s*(\d+)\|/) { my $lnn = $1; my @columns = split(/\s*\|\s*/, $line); foreach (@columns) { if (/([\d\.]+)%/) { $NODE_INFO{$lnn}{'Percent Used'} = $1; debug("NODE_INFO->${lnn}->Percent Used=$1"); last; } } $NODE_INFO{$lnn}{'SSD Storage'} = $columns[$#columns]; debug("NODE_INFO->${lnn}->SSD Storage=$NODE_INFO{$lnn}{'SSD Storage'}"); } } } # End gatherStatus ### # gatherSyncIQ: Gathers SyncIQ policies and source and target records. sub gatherSyncIQ { return if wasCalled(); gatherSyncIQTarget(); gatherSIQPoliciesGC(); gatherSyncPoliciesList(); } # End gatherSyncIQ ### # gatherSyncIQTarget: Gather SyncIQ target information from record files sub gatherSyncIQTarget { return if wasCalled(); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 grep -E '<(policy-name|src-cluster-name|job-status-timestamp|id|target-dir)>' /ifs/.ifsvar/modules/tsm/config/target_record.xml /ifs/.ifsvar/modules/tsm/config/target_records/*]); } else { @gather = gatherLocal(qq[2>&1 grep -E '<(policy-name|src-cluster-name|job-status-timestamp|id|target-dir)>' local/ifsvar_modules.tar/modules/tsm/config/target_record.xml local/ifsvar_modules.tar/modules/tsm/config/target_records/*]); } my $pid = ""; my %siq_incoming = (); foreach (@gather) { $pid = $1 if (/(.*?)<\/id>/); next unless $pid; if (/(.*?)<\/policy-name>/) { $siq_incoming{$pid}{'Policy Name'} = $1; } elsif (/(.*?)<\/src-cluster-name>/) { $siq_incoming{$pid}{'Source Cluster'} = $1; } elsif (/(.*?)<\/job-status-timestamp>/) { $siq_incoming{$pid}{'Timestamp'} = $1; } elsif (/(.*?)<\/target-dir>/) { $siq_incoming{$pid}{'Target Directory'} = $1; } } foreach my $pid (keys %siq_incoming) { next unless $siq_incoming{$pid}{'Source Cluster'} and $siq_incoming{$pid}{'Timestamp'} and $siq_incoming{$pid}{'Policy Name'} and $RPS{'Timestamp'} - $siq_incoming{$pid}{'Timestamp'} < 7776000; foreach my $key (keys %{$siq_incoming{$pid}}) { next if $key eq 'Policy ID'; $CLUSTER_INFO{'SyncIQ'}{'Target'}{$pid}{$key} = $siq_incoming{$pid}{$key}; debug("CLUSTER_INFO->SyncIQ->Target->${pid}->${key}=$siq_incoming{$pid}{$key}"); } } } # End gatherSyncIQTarget ### # gatherSyncPoliciesList: Check SyncIQ policy information sub gatherSyncPoliciesList { return if wasCalled(); my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal(qq[2>&1 ${SUDO} isi sync pol list -v | grep -E 'ID:|Name:|Enabled:|Target:|Source Subnet:|Source Pool:|Last Job State:|Last Started:|Report']); } else { @gather = gatherLocal(qq[2>&1 cat */isi_sync_policy | grep -E 'ID:|Name:|Enabled:|Target:|Source Subnet:|Source Pool:|Last Job State:|Last Started:|Report']); } debug(@gather) if @gather; # Gather information my $id = ""; my %siq_policies = (); my $restrict_by_subnet = ""; foreach (@gather) { if (/^\s*ID:\s+(.*)$/) { $id = $1; last if $siq_policies{$id}; } elsif (/^\s*Name:\s+(.*)$/) { $siq_policies{$id}{'Name'} = $1; } elsif (/^\s*Enabled:\s+(Yes)$/) { $siq_policies{$id}{'Enabled'} = $1; } elsif (/^\s*Target:\s+(.*)$/) { $siq_policies{$id}{'Target'} = $1; } elsif (/^\s*Source Subnet:\s+(.*)$/) { $restrict_by_subnet = $1 if $1 !~ /^-$/; } elsif (/^\s*Source Pool:\s+(.*)$/) { $siq_policies{$id}{'Restrict Source Nodes'} = "$restrict_by_subnet:$1" if $1 !~ /^-$/; } elsif (/^\s*Last Job State:\s+(.*)$/) { $siq_policies{$id}{'Last Job State'} = $1; } elsif (/^\s*Last Started:\s+(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)$/) { $siq_policies{$id}{'Timestamp'} = timelocal($6, $5, $4, $3, $2 - 1, $1); } elsif (/^\s*Report Max Age:\s+(\S+)$/) { $siq_policies{$id}{'Max Age'} = convertDurationToSeconds($1); } elsif (/^\s*Report Max Count:\s+(\d+)$/) { $siq_policies{$id}{'Max Count'} = $1; } } # Assign information by Policy ID for each policy to CLUSTER_INFO foreach my $pid (keys %siq_policies) { foreach my $key (keys %{$siq_policies{$pid}}) { $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{$key} = $siq_policies{$pid}{$key}; debug("CLUSTER_INFO->SyncIQ->Source->${pid}->${key}=$siq_policies{$pid}{$key}"); } } } # End gatherSyncPoliciesList ### # gatherUpgradeCluster: Gathers information from isi upgrade cluster view. sub gatherUpgradeCluster { return if wasCalled(); return unless compareVersions($CLUSTER_INFO{'OneFS'}, "8.0") >= 0; my @gather = (); if ($RPS{'Live'}) { @gather = gatherLocal("2>&1 ${SUDO} isi upgrade cluster view"); } else { @gather = gatherLocal("2>&1 cat */isi_upgrade_cluster_view local/isi_upgrade_view"); } foreach my $line (@gather) { if ($line =~ /Cluster Upgrade State: (.*)/) { next if $CLUSTER_INFO{'Cluster Upgrade State'}; $CLUSTER_INFO{'Cluster Upgrade State'} = $1; debug("CLUSTER_INFO->Cluster Upgrade State=$1") } elsif ($line =~ /Current Upgrade Activity: (.*)/) { next if $CLUSTER_INFO{'Current Upgrade Activity'}; $CLUSTER_INFO{'Current Upgrade Activity'} = $1; debug("CLUSTER_INFO->Current Upgrade Activity=$1"); } } } # End gatherUpgradeCluster ### # gatherUptime: Gathers and calculates each nodes uptime sub gatherUptime { return if wasCalled(); # Gather information my %gather = (); if ($RPS{'Live'}) { %gather = gatherCluster("2>&1 ${SUDO} isi_for_array uptime"); } else { %gather = gatherCluster("2>&1 grep -H . */uptime"); } # uptime_gather_adjustment is the time difference between the log gather and current time my $uptime_gather_adjustment = 0; $uptime_gather_adjustment = $RPS{'Runtime'} - $RPS{'Timestamp'}; debug("gatherUptime: uptime_gather_adjustment=$uptime_gather_adjustment seconds"); # Anaylze information foreach my $lnn (@LNNS) { if (!$gather{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$lnn}}) { $NODE_INFO{$lnn}{'uptime_gathered'} = $line; debug("NODE_INFO->${lnn}->uptime_gathered=$line"); # Gather current time if ($line =~ /(\d+:\d+[AP]M)/) { $NODE_INFO{$lnn}{'time'} = $1; debug("NODE_INFO->${lnn}->time->$NODE_INFO{$lnn}{'time'}"); } # uptime_seconds will hold the number of seconds a node has been up for my $uptime_seconds = 0; if ($line =~ /\s(\d+)\s+day/) { $uptime_seconds += $1 * 86400; } if ($line =~ /\s(\d+)\s+hr/) { $uptime_seconds += $1 * 3600; } if ($line =~ /\s(\d+)\s+min/) { $uptime_seconds += $1 * 60; } if ($line =~ /\s(\d+)\s+sec/) { $uptime_seconds += $1; } if ($line =~ /\s(\d+):(\d+),/) { $uptime_seconds += $1 * 3600 + $2 * 60; } # If the uptime_seconds is 0, there was no uptime information found. next unless $uptime_seconds; # Total uptime is the number of seconds of reported uptime plus the adjustment variable if running on a logset $NODE_INFO{$lnn}{'uptime'} = $uptime_seconds + $uptime_gather_adjustment; debug("NODE_INFO->${lnn}->uptime->$NODE_INFO{$lnn}{'uptime'}"); } } } # End gatherUptime ### # hasLicense: Returns 1 if licensed, otherwise 0 sub hasLicense { # Gather license details incase not previously gathered gatherLicense(); # Return 0 if license details were not gathered for whatever reason unless ($CLUSTER_INFO{'License'}) { debug("hasLicense: Returning 0 (No license details available)"); return 0; } # Return 0 if there is no license being checked unless (@_ and $_[0]) { debug("hasLicense: Returning 0 (No arguments provided)"); return 0; } # Search for the licensed feature debug("hasLicense: Looking for feature that contains: $_[0]"); my ($feature) = grep {$_ =~ /$_[0]/i} keys %{$CLUSTER_INFO{'License'}}; # Return 0 if unable to locate licensed feature unless ($feature and $CLUSTER_INFO{'License'}{$feature} and $CLUSTER_INFO{'License'}{$feature}{'Status'}) { debug("hasLicense: Returning 0 (No feature found)"); return 0; } debug("hasLicense: Found: $feature"); # Return 1 if licensed if ($CLUSTER_INFO{'License'}{$feature}{'Status'} =~ /^(activated|licensed|evaluation)$/i) { debug("hasLicense: Returning 1 ($feature is $CLUSTER_INFO{'License'}{$feature}{'Status'})"); return 1; } # Return 0 if not licensed debug("hasLicense: Returning 0 ($feature is $CLUSTER_INFO{'License'}{$feature}{'Status'})"); return 0; } # End hasLicense ### # getActivitiesName: Returns an array of activity names that # corresponds to activities in the RCM Scheduler based on the # activity type ( Drive Firmware/ Node Firmware/ Rolling # Upgrade/ Rolling Reboot/ FCO F042415EE ) and a number of # nodes or for Isilon Hourly activities the duration. sub getActivitiesName { return unless $_[0]; our %Activities; my $activity_type = $_[0]; my @activities = (); $activity_type = "Rolling Reboot" if ($activity_type eq "Proactive Reboot" or $activity_type eq "Drive Firmware Reboot"); $activity_type = "Rolling Node Upgrade" if ($activity_type eq "Rolling OneFS"); $activity_type = "Simultaneous Reboot (Offline)" if ($activity_type eq "Simultaneous Reboot"); $activity_type = "Simultaneous (Offline) Upgrade" if ($activity_type eq "Simultaneous OneFS"); $activity_type = "Node Firmware" if ($activity_type eq "BMC Firmware"); $activity_type = "Drive Firmware" if ($activity_type eq "Drive Support Package" or $activity_type eq "Drive Firmware Package"); return ($activity_type) unless $Activities{$activity_type}; return unless $_[1]; my $count = $_[1]; my $last_index = 0; foreach my $index (sort {$b <=> $a} keys %{$Activities{$activity_type}}) { last if $count <= 0; if ($count > $index) { $index = $last_index if $last_index; push(@activities, $Activities{$activity_type}{$index}); $count = $count - $index; $last_index = 0; redo; } elsif ($count == $index) { push(@activities, $Activities{$activity_type}{$index}); $last_index = 0; last; } $last_index = $index; } push(@activities, $Activities{$activity_type}{$last_index}) if ($last_index); return @activities; } # End getActivitiesName ### # Provided the # of the activity in the upgrade plan, return the # estimated duration for that activity. sub getActivityDuration { return unless $_[0]; our %Durations; my %activity = %{$_[0]}; my $duration = 0; # For hourly activities, return duration in minutes if ($activity{'Type'} eq "Isilon Hourly") { return $activity{'Count'} * 60; } # Add the base duration if ($Durations{$activity{'Type'}}{'Base'}) { $duration += $Durations{$activity{'Type'}}{'Base'}; } # Add per node/drive estimate if ($activity{'Type'} eq 'Drive Support Package' and not $activity{'Current Code'} =~ /^7\.1\.1\.[0123]$/ and not $activity{'Current Code'} =~ /^7\.2\.0\.[01]$/) { $duration += $RPS{$activity{'Type'}}{'Drives'} * $Durations{$activity{'Type'}}{'Drives'}; } elsif ($Durations{$activity{'Type'}}{'Nodes'}) { $duration += $activity{'Count'} * $Durations{$activity{'Type'}}{'Nodes'}; } # Add additional time based on hardware type for reboots if ($activity{'Nodes'} and $activity{'Type'} =~ /Rolling|Node Firmware|BMC Firmware/) { foreach my $lnn (@{$activity{'Nodes'}}) { next unless $NODE_INFO{$lnn}{'Product'}; my ($match) = grep {$NODE_INFO{$lnn}{'Product'} =~ /$_/} keys %{$Durations{'Reboot'}}; next unless $match; $duration += $Durations{'Reboot'}{$match}; } } # Add additional time based on hardware type for node firmware updates if ($activity{'Nodes'} and $activity{'Type'} eq 'Node Firmware') { foreach my $lnn (@{$activity{'Nodes'}}) { next unless $NODE_INFO{$lnn}{'Product'}; my ($match) = grep {$NODE_INFO{$lnn}{'Product'} =~ /$_/} keys %{$Durations{$activity{'Type'}}}; next unless $match; $duration += $Durations{$activity{'Type'}}{$match}; } } # Add additional time based on the devices for node firmware updates if ($activity{'Nodes'} and $activity{'Type'} eq 'Node Firmware' and $RPS{$activity{'Type'}}{'Devices'}) { foreach my $device (keys %{$RPS{$activity{'Type'}}{'Devices'}}) { my ($match) = grep {$device =~ /$_/} keys %{$Durations{$activity{'Type'}}}; next unless $match; $duration += $Durations{$activity{'Type'}}{$match} * scalar(@{$RPS{$activity{'Type'}}{'Devices'}{$device}}); } } # Add additional time based on the number of patches for rebootless patch install activities if ($activity{'Patches'}) { $duration += $Durations{'Patch Install (Rebootless)'}{'Patch'} * scalar(@{$activity{'Patches'}}); } return $duration; } # End getActivityDuration ### # isUpgradeRestripe: Provided a source and target version, returns 1 if after the upgrade a restripe job will run sub isUpgradeRestripe { if (!(defined $_[0] and defined $_[1])) { debug("isUpgradeRestripe: Not enough arguments provided."); return 0; } debug("isUpgradeRestripe: $_[0] $_[1]"); return 1 if (compareVersions($_[0], "5") == 0 and compareVersions($_[1], "6") >= 0); return 1 if (compareVersions($_[0], "6") == 0 and compareVersions($_[1], "7") >= 0); return 1 if (compareVersions($_[0], "7") == 0 and compareVersions($_[1], "8") >= 0); return 0; } # End isUpgradeRestripe ### # isUpgradeSimultaneous: Provided a source and target version, returns 1 if upgrade must be simultaneous sub isUpgradeSimultaneous { if (!(defined $_[0] and defined $_[1])) { debug("isUpgradeSimultaneous: Not enough arguments provided."); return 0; } debug("isUpgradeSimultaneous: $_[0] $_[1]"); return 1 if (compareVersions($_[0], "5") == 0 and compareVersions($_[1], "6") >= 0); return 1 if (compareVersions($_[0], "6.0") == 0 and compareVersions($_[1], "6.5") >= 0); return 1 if (compareVersions($_[0], "6.5") == 0 and compareVersions($_[1], "7") >= 0); return 1 if (compareVersions($_[0], "7.0") == 0 and compareVersions($_[1], "7.1") >= 0); return 1 if (compareVersions($_[0], "7") == 0 and compareVersions($_[1], "8") >= 0); return 1 if ($_[0] =~ /^7\.1\.1\.[89]$/ and $_[1] =~ /^7\.2\.1\.3$/); return 0; } # End isUpgradeSimultaneous ### # patchSort: Sorting routine to allow patch ordering when a patch has a priority. sub patchSort { if ($RPS{$_}{$a}{'Priority'}) { if ($RPS{$_}{$b}{'Priority'}) { return $RPS{$_}{$b}{'Priority'} - $RPS{$_}{$a}{'Priority'}; } else { return -1; } } elsif ($RPS{$_}{$b}{'Priority'}) { return 1; } else { return $b cmp $a; } } # End patchSort ### # printCustomerInformation: Prints gathered customer information sub printCustomerInformation { return unless $RPS{'Customer Information'}; print "\n"; print colored("\nThe following information should be included to the customer:", 'bold underline'); print "\n"; if ($RPS{'Customer Information'}{'Generic'}) { print "\n$_" foreach (@{$RPS{'Customer Information'}{'Generic'}}); } foreach my $header (keys %{$RPS{'Customer Information'}}) { next if $header eq "Generic"; print "\n--------------------"; print "\n $header"; print "\n--------------------"; print "\n$_" foreach (@{$RPS{'Customer Information'}{$header}}); } print "\n"; } # End printCustomerInformation ### # printRequiredFiles: Prints the list of files required. sub printRequiredFiles { return unless $RPS{'Required Files'}; our %CodeLinks; print "\n"; print colored("\nThe following files will need to be downloaded to /ifs/data/:", 'bold underline'); print "\n"; foreach my $file (sort @{$RPS{'Required Files'}}) { print "\n$file"; print "\nCode: $CodeLinks{$file}{'Code'}" if $CodeLinks{$file}{'Code'}; print "\nsha256: $CodeLinks{$file}{'sha256'}" if $CodeLinks{$file}{'sha256'}; print "\n"; } } # End printRequiredFiles ### # printSystemDetails: Prints system details sub printSystemDetails { print "\nSystem Details:"; print "\nCluster Name: $CLUSTER_INFO{'Name'}" if $CLUSTER_INFO{'Name'}; print "\nNumber of Nodes: $CLUSTER_INFO{'Node Count'}" if $CLUSTER_INFO{'Node Count'}; print "\nReplication Status (SyncIQ): ".(($CLUSTER_INFO{'SyncIQ'}{'Source'} || $CLUSTER_INFO{'SyncIQ'}{'Target'})?"Yes":"No").""; print "\nCompliance Mode: ".($CLUSTER_INFO{'Compliance'}?"Yes":"No").""; print "\nCurrent OneFS: $CLUSTER_INFO{'OneFS'}" if $CLUSTER_INFO{'OneFS'}; print "\nDestination OneFS: $RPS{'Target'}" if $RPS{'Target'}; } # End printSystemDetails ### # upgradePlan: Steps used to generate and print an upgrade plan sub upgradePlan { return if wasCalled(); checkUptime(); checkNodeFirmware(); checkDriveFirmware(); checkUpgradePath(); checkPatches(); @{$RPS{'Upgrade Order'}} = upgradePlanOrder(); @{$RPS{'Upgrade Plan'}} = upgradePlanGenerator(@{$RPS{'Upgrade Order'}}); @{$RPS{'Hourly Plan'}} = upgradePlanConvertToHourly($RPS{'Upgrade Plan'}); @{$RPS{'Combined Plan'}} = upgradePlanConvertToCombined($RPS{'Upgrade Plan'}); if ($RPS{'Upgrade Plan'}) { my $activity_number = 1; my $title = "Overview of Upgrade Steps"; foreach my $act_href (@{$RPS{'Upgrade Plan'}}) { my %activity = %{$act_href}; next unless $activity{'Type'}; my $description = $activity{'Description'} || $activity{'Type'}; my $duration = getActivityDuration($act_href); addCustomerInformation ($title, "$activity_number: $description (". sprintf("%.1f", $duration/60)." hours)"); addCustomerInformation ($title, "Current Code: $activity{'Current Code'}") if $activity{'Current Code'}; addCustomerInformation ($title, "Upgrading To: $activity{'Target'}") if $activity{'Target'}; addCustomerInformation ($title, "Patch: ".join(", ", @{$activity{'Patches'}})) if $activity{'Patches'}; addCustomerInformation ($title, "Nodes: ".compressRange(@{$activity{'Nodes'}}). " (".scalar(@{$activity{'Nodes'}})." nodes)") if $activity{'Nodes'}; if ($activity{'Additional Comments'}) { if (scalar(@{$activity{'Additional Comments'}}) == 1) { addCustomerInformation ($title, "Additional Comments: $activity{'Additional Comments'}[0]"); } else { addCustomerInformation ($title, "Additional Comments: \n# ".join("\n# ", @{$activity{'Additional Comments'}})); } } addCustomerInformation($title, ""); $activity_number++; } } printCustomerInformation(); printRequiredFiles(); if ($RPS{'F011018EE'} or $RPS{'F121917EE'} or (scalar(@{$RPS{'Combined Plan'}}) == 1 and $RPS{'Combined Plan'}[0]{'Count'} <= 12)) { upgradePlanPrint(@{$RPS{'Combined Plan'}}); } else { upgradePlanPrint(@{$RPS{'Upgrade Plan'}}); #upgradePlanPrint(@{$RPS{'Hourly Plan'}}); } #upgradePlanPrint(@{$RPS{'Upgrade Plan'}}); #upgradePlanPrint(@{$RPS{'Hourly Plan'}}); #upgradePlanPrint(@{$RPS{'Combined Plan'}}); } # End upgradePlan ### # upgradePlanConvertToCombined: Given an upgrade plan will convert it # into a plan that is hourly activities and combined sub upgradePlanConvertToCombined { return () unless $_[0]; my @c_plan = (); my $c_duration = 0; my $c_href = {()}; $,="\n"; foreach my $a_href (@{$_[0]}) { my $duration = getActivityDuration($a_href); if ($a_href->{'Type'} eq "Isilon Hourly" or $a_href->{'Standalone'}) { push(@c_plan, $c_href) if %$c_href; push(@c_plan, $a_href); $c_duration = 0; $c_href = {()}; next; } elsif ($duration + $c_duration < 720) { if (%$c_href) { push(@{$c_href->{'Additional Comments'}}, ""); $c_href->{'Count'} += $duration/60; $c_duration += $duration; } else { $c_href->{'Type'} = "Isilon Hourly"; $c_href->{'Count'} = $duration/60; $c_duration = $duration; } } else { push(@c_plan, $c_href) if %$c_href; $c_href = {()}; $c_duration = $duration; $c_href->{'Type'} = "Isilon Hourly"; $c_href->{'Count'} = $duration/60; } if ($a_href->{'Type'}) { my ($type) = getActivitiesName($a_href->{'Type'}, $a_href->{'Count'}); push(@{$c_href->{'Additional Comments'}}, "Activity Type: $type (". sprintf("%.1f", $duration/60)." hours)"); } if ($a_href->{'Current Code'}) { push(@{$c_href->{'Additional Comments'}}, "Current Code: $a_href->{'Current Code'}"); } if ($a_href->{'Target'}) { push(@{$c_href->{'Additional Comments'}}, "Upgrading To: $a_href->{'Target'}"); } if ($a_href->{'Patches'}) { push(@{$c_href->{'Additional Comments'}}, "Patch: ".join(", ", @{$a_href->{'Patches'}})); } if ($a_href->{'Additional Comments'}) { push(@{$c_href->{'Additional Comments'}}, "Additional Details: " .join("\n# ", @{$a_href->{'Additional Comments'}})); } } push(@c_plan, $c_href) if %$c_href; return @c_plan; } # End upgradePlanConvertToCombined ### # upgradePlanConvertToHourly: Given an upgrade plan will convert it # into a plan that is hourly activities as opposed typed activities. sub upgradePlanConvertToHourly { return () unless $_[0]; my @hourly_plan = (); foreach my $a_href (@{$_[0]}) { my $duration = getActivityDuration($a_href)/60; while ($duration > 0) { my $h_href = {()}; $h_href->{'Type'} = "Isilon Hourly"; if ($duration > 12) { $h_href->{'Count'} = 12; } else { $h_href->{'Count'} = $duration; } $duration = $duration - 12; push(@{$h_href->{'Additional Comments'}}, "Activity Type: $a_href->{'Type'}") if ($a_href->{'Type'}); push(@{$h_href->{'Additional Comments'}}, "Current Code: $a_href->{'Current Code'}") if ($a_href->{'Current Code'}); push(@{$h_href->{'Additional Comments'}}, "Upgrading To: $a_href->{'Target'}") if ($a_href->{'Target'}); push(@{$h_href->{'Additional Comments'}}, "Patch: ".join(", ", @{$a_href->{'Patches'}})) if ($a_href->{'Patches'}); push(@{$h_href->{'Additional Comments'}}, "Extra Instructions: ",@{$a_href->{'Additional Comments'}}) if ($a_href->{'Additional Comments'}); push(@hourly_plan, $h_href); } } return @hourly_plan; } # End upgradePlanConvertToHourly ### # upgradePlanGenerator: Generates the upgrade plan given the upgrade order is set sub upgradePlanGenerator { return unless @_; my @upgrade_order = @_; my @upgrade_plan = (); my $current_code = $CLUSTER_INFO{'OneFS'}; foreach(@upgrade_order) { if (/Support|Firmware|Reboot/) { addRequiredFiles(@{$RPS{$_}{'Files'}}) if $RPS{$_}{'Files'}; my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Type'} = $_; if (/Drive Firmware Reboot/ or /Proactive Reboot/) { $activity{'Type'} = "Rolling Reboot"; $activity{'Type'} = "Simultaneous Reboot" if $RPS{'Simultaneous'}; } $activity{'Target'} = $RPS{$_}{'Target'} if $RPS{$_}{'Target'}; @{$activity{'Patches'}} = keys %{$RPS{$_}{'Patch'}} if $RPS{$_}{'Patch'}; if ($RPS{$_}{'Nodes'}) { $activity{'Count'} = scalar(@{$RPS{$_}{'Nodes'}}); @{$activity{'Nodes'}} = @{$RPS{$_}{'Nodes'}}; } else { $activity{'Count'} = $CLUSTER_INFO{'Node Count'}; @{$activity{'Nodes'}} = keys %NODE_INFO; } $activity{'Additional Comments'} = $RPS{$_}{'Additional Comments'}; push(@upgrade_plan, \%activity); if ($RPS{$_}{'Priority'}) { addCustomerInformation("Upgrade Plan", @{$RPS{$_}{'Priority'}}); } if ($RPS{$_}{'Patch'}) { foreach my $patch (keys %{$RPS{$_}{'Patch'}}) { addRequiredFiles($patch) unless $patch =~ /uninstall/; addCustomerInformation($RPS{$_}{'Patch'}{$patch}{'Impacts'}) if $RPS{$_}{'Patch'}{$patch}{'Impacts'}; addCustomerInformation($RPS{$_}{'Patch'}{$patch}{'Important'}) if $RPS{$_}{$patch}{'Important'}; } } } # OneFS Upgrade elsif (/OneFS/) { foreach my $target (sort {compareVersions($a, $b)} @{$RPS{'Upgrade Path'}}) { next if $target eq $current_code; addRequiredFiles("Isilon OneFS $target installation file"); my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Type'} = "Rolling OneFS"; $activity{'Type'} = "Simultaneous OneFS" if (isUpgradeSimultaneous($current_code, $target) or $RPS{'Simultaneous'}); $activity{'Target'} = $target; $activity{'Count'} = $CLUSTER_INFO{'Node Count'}; @{$activity{'Nodes'}} = keys %NODE_INFO; if ($RPS{'Target'} eq $target and $RPS{'Post Upgrade Patches'}) { @{$activity{'Additional Comments'}} = ("Subsequent patch installs should be scheduled in the same maintenance window as the OneFS upgrade"); addCustomerInformation("Upgrade Plan", "Patch installs are recommended to be performed in the same maintenance window as the OneFS upgrade"); } push(@upgrade_plan, \%activity); if ($target ne $RPS{'Target'} and isUpgradeRestripe($current_code, $target)) { my %activity = (); $activity{'Type'} = "One week delay to allow 'Upgrade' data restriping job to complete."; $activity{'Standalone'} = 1; push(@upgrade_plan, \%activity); } $current_code = $target; } } # Patches elsif (/Patches/) { my @patches_without_reboot = (); my @patches_with_reboot = (); my @patches_with_standalone_rolling_reboot = (); my @reboot_nodes = (); my $simultaneous = 0; # Check if any patch requires a simultaneous reboot foreach my $patch (keys %{$RPS{$_}}) { $simultaneous = 1 if ($RPS{$_}{$patch}{'Reboot'} and $RPS{$_}{$patch}{'Reboot'} eq "Simultaneous"); } @reboot_nodes = @LNNS if $simultaneous; # Put together which nodes require rebooting and sort patches into patches with reboots and patches without reboots foreach my $patch (sort patchSort keys %{$RPS{$_}}) { addRequiredFiles($patch) unless $patch =~ /uninstall/; addCustomerInformation($RPS{$_}{$patch}{'Important'}) if $RPS{$_}{$patch}{'Important'}; # The patch requires a service restart and can be done rolling to avoid this impact if (!($simultaneous or $RPS{'Simultaneous'}) and $RPS{$_}{$patch}{'Service Restart'}) { push(@patches_with_standalone_rolling_reboot, $patch) unless (grep {$_ eq $patch} @patches_with_standalone_rolling_reboot); } # The patch requires a manual reboot elsif ($RPS{$_}{$patch}{'Reboot'}) { addCustomerInformation($RPS{$_}{$patch}{'Impacts'}) if $RPS{$_}{$patch}{'Impacts'}; push(@patches_with_reboot, $patch); if (!$simultaneous) { foreach my $lnn (@{$RPS{$_}{$patch}{'Reboot'}}) { push(@reboot_nodes, $lnn) unless (grep {$_ eq $lnn} @reboot_nodes); } } } # The patch does not require a reboot else { addCustomerInformation($RPS{$_}{$patch}{'Impacts'}) if $RPS{$_}{$patch}{'Impacts'}; push(@patches_without_reboot, $patch); } } if (@patches_with_reboot or @patches_without_reboot) { # Previous step is an upgrade to OneFS if (scalar(@upgrade_plan) and $upgrade_plan[$#upgrade_plan] and $upgrade_plan[$#upgrade_plan]{'Type'} =~ /OneFS/) { # Add rebootless patches to previous activity if (@patches_without_reboot) { push(@{$upgrade_plan[$#upgrade_plan]{'Patches'}}, @patches_without_reboot); } # Add rebootfull patches as a separate activity if (@patches_with_reboot) { my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Type'} = "Rolling Reboot"; $activity{'Type'} = "Simultaneous Reboot" if $simultaneous or $RPS{'Simultaneous'}; $activity{'Description'} = "Patch Install (Rolling Reboot)"; $activity{'Description'} = "Patch Install (Simultaneous Reboot)" if $simultaneous or $RPS{'Simultaneous'}; $activity{'Count'} = scalar(@reboot_nodes); @{$activity{'Nodes'}} = @reboot_nodes; @{$activity{'Patches'}} = @patches_with_reboot; @{$activity{'Additional Comments'}} = ("Only the following nodes require rebooting: ".compressRange(@reboot_nodes)) if (scalar(@reboot_nodes) < $CLUSTER_INFO{'Node Count'}); push(@upgrade_plan, \%activity); } } # Previous step is a proactive reboot elsif (scalar(@upgrade_plan) and $upgrade_plan[$#upgrade_plan] and $upgrade_plan[$#upgrade_plan]{'Type'} =~ /Proactive Reboot|Rolling Reboot/) { # Add rebootless and rebootfull patches to previous activity push(@{$upgrade_plan[$#upgrade_plan]{'Patches'}}, @patches_without_reboot, @patches_with_reboot); } # Previous step is not going to be used else { if (@patches_with_reboot) { my %activity = (); $activity{'Current Code'} = $current_code; if (@reboot_nodes) { $activity{'Type'} = "Rolling Reboot"; $activity{'Type'} = "Simultaneous Reboot" if $simultaneous or $RPS{'Simultaneous'}; $activity{'Description'} = "Patch Install (Rolling Reboot)"; $activity{'Description'} = "Patch Install (Simultaneous Reboot)" if $simultaneous or $RPS{'Simultaneous'}; $activity{'Count'} = scalar(@reboot_nodes); @{$activity{'Nodes'}} = @reboot_nodes; } @{$activity{'Patches'}} = merge(@patches_with_reboot, @patches_without_reboot); @{$activity{'Additional Comments'}} = ("Only the following nodes require rebooting: ".compressRange(@reboot_nodes)) if (@reboot_nodes and scalar(@reboot_nodes) < $CLUSTER_INFO{'Node Count'}); push(@upgrade_plan, \%activity); } else { my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Type'} = "Patch Install (Rebootless)"; @{$activity{'Patches'}} = @patches_without_reboot; push(@upgrade_plan, \%activity); } } } foreach my $patch (@patches_with_standalone_rolling_reboot) { my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Type'} = "Rolling Reboot"; $activity{'Description'} = "Patch Install (Rolling Reboot)"; $activity{'Count'} = $CLUSTER_INFO{'Node Count'}; @{$activity{'Nodes'}} = keys %NODE_INFO; @{$activity{'Patches'}} = ($patch); if ($_ eq "Deprecated Patches") { @{$activity{'Additional Comments'}} = ("$patch with --rolling=true"); } else { @{$activity{'Additional Comments'}} = ("Apply $patch with --rolling=true"); } push(@upgrade_plan, \%activity); } } # FCO F042415EE elsif (/FCO F042415EE/) { addRequiredFiles("Isilon Drive Firmware Package 1.10"); my @nodes_with_l3 = (); foreach my $lnn (@{$RPS{'FCO F042415EE'}{'Nodes'}}) { if ($NODE_INFO{$lnn}{'L3'}) { push(@nodes_with_l3, $lnn); } else { my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Standalone'} = 1; $activity{'Type'} = "FCO F042415EE"; $activity{'Count'} = 1; $activity{'Target'} = "Drive Firmware Package 1.10"; @{$activity{'Additional Comments'}} = ("Node $lnn"); push(@upgrade_plan, \%activity); } } if (@nodes_with_l3) { my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Standalone'} = 1; $activity{'Type'} = "FCO F042415EE"; $activity{'Count'} = scalar(@nodes_with_l3); $activity{'Target'} = "Drive Firmware Package 1.10"; @{$activity{'Additional Comments'}} = ("Additional Comments: Nodes ".compressRange(@nodes_with_l3)) if (scalar(@nodes_with_l3) != scalar(@{$RPS{'FCO F042415EE'}{'Nodes'}}) or scalar(@{$RPS{'FCO F042415EE'}{'Nodes'}}) != $CLUSTER_INFO{'Node Count'}); push(@upgrade_plan, \%activity); } } # FCO F011018EE elsif (/F011018EE/) { my %activity = (); $activity{'Current Code'} = $current_code; $activity{'Type'} = "Isilon Hourly"; $activity{'Count'} = 1; if (/F011018EE1/) { @{$activity{'Additional Comments'}} = ("Start IntegrityScan job"); } elsif (/F011018EE2/) { @{$activity{'Additional Comments'}} = ("Review IntegrityScan job results, modify sysctl setting, and start Collect job"); } elsif (/F011018EE3/) { @{$activity{'Additional Comments'}} = ("Review Collect job results and restore sysctl setting"); } push(@upgrade_plan, \%activity); } } return @upgrade_plan; } # End upgradePlanGenerator ### # upgradePlanOrder: Returns the order of activites to be performed sub upgradePlanOrder { # Just incase node firmware was flagged to be performed post OneFS, do BMC firmware updates post node firmware if ($RPS{'BMC Firmware'}{'Position'} and $RPS{'BMC Firmware'}{'Position'} == 1 and $RPS{'CMC Firmware'}{'Nodes'} and $RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == 2) { $RPS{'BMC Firmware'}{'Position'} = 2; } # Flag Node firmware updates as optional unless they are at FCO F010517EE levels if ($RPS{'F010517EE'} and !$RPS{'FCO'}{'F010517EE'}{'Nodes'}) { addCustomerInformation("Upgrade Plan", "Node firmware updates available but already at required levels for FCO F010517EE and are considered optional."); } # Flag Node Firmware updates as optional unless they are a priority if ($RPS{'T030518EE'} and $RPS{'Node Firmware'}{'Position'} and !$RPS{'Node Firmware'}{'Priority'}) { addCustomerInformation("Upgrade Plan", "Node firmware updates available and included but considered optional for TSE T030518EE."); } if ($RPS{'F033018EE'} and $RPS{'Node Firmware'}{'Position'} and !$RPS{'Node Firmware'}{'Priority'}) { addCustomerInformation("Upgrade Plan", "Node firmware updates available and included but considered optional for FCO F033018EE."); } if ($RPS{'F082318EE'} and $RPS{'Drive Support Package'}{'Position'}) { addCustomerInformation("Upgrade Plan", "Drive support package updates available and included but considered optional for FCO F082318EE."); } # Define/reset the upgrade order my @upgrade_order = (); push(@upgrade_order, "Drive Support Package") if ($RPS{'Drive Support Package'}{'Position'} and $RPS{'Drive Support Package'}{'Position'} == "1"); push(@upgrade_order, "Drive Firmware Package") if ($RPS{'Drive Firmware Package'}{'Position'} and $RPS{'Drive Firmware Package'}{'Position'} == "1"); push(@upgrade_order, "Drive Firmware Reboot") if ($RPS{'Drive Firmware Reboot'}{'Position'} and $RPS{'Drive Firmware Reboot'}{'Position'} == "1"); push(@upgrade_order, "Proactive Reboot") if ($RPS{'Proactive Reboot'}); push(@upgrade_order, "Deprecated Patches") if ($RPS{'Deprecated Patches'}); push(@upgrade_order, "Pre Upgrade Patches") if ($RPS{'Pre Upgrade Patches'}); push(@upgrade_order, "Node Firmware") if ($RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == "1"); push(@upgrade_order, "BMC Firmware") if ($RPS{'BMC Firmware'}{'Position'} and $RPS{'BMC Firmware'}{'Position'} == "1"); push(@upgrade_order, "Drive Support Package") if ($RPS{'Drive Support Package'}{'Position'} and $RPS{'Drive Support Package'}{'Position'} == "2"); push(@upgrade_order, "Drive Firmware Reboot") if ($RPS{'Drive Firmware Reboot'}{'Position'} and $RPS{'Drive Firmware Reboot'}{'Position'} == "2"); push(@upgrade_order, "OneFS") if ($RPS{'Upgrade Path'} and scalar(@{$RPS{'Upgrade Path'}}) > 1); push(@upgrade_order, "Post Upgrade Patches") if ($RPS{'Post Upgrade Patches'}); push(@upgrade_order, "Node Firmware") if ($RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == "2"); push(@upgrade_order, "BMC Firmware") if ($RPS{'BMC Firmware'}{'Position'} and $RPS{'BMC Firmware'}{'Position'} == "2"); push(@upgrade_order, "Drive Support Package") if ($RPS{'Drive Support Package'}{'Position'} and $RPS{'Drive Support Package'}{'Position'} == "3"); push(@upgrade_order, "Drive Firmware Reboot") if ($RPS{'Drive Firmware Reboot'}{'Position'} and $RPS{'Drive Firmware Reboot'}{'Position'} == "3"); push(@upgrade_order, "FCO F042415EE") if ($RPS{'FCO F042415EE'}{'Nodes'}); # Only Node Firmware updates for FCO F121917EE if ($RPS{'F121917EE'}) { @upgrade_order = (); @upgrade_order = ("Node Firmware") if $RPS{'FCO'}{'F121917EE'}{'Nodes'} and $RPS{'Node Firmware'}{'Position'}; } # Special plan for FCO F011018EE if ($RPS{'F011018EE'}) { if ($RPS{'FCO'}{'F011018EE'}{'Nodes'} and $RPS{'Node Firmware'}{'Position'}) { @upgrade_order = ("Node Firmware", "F011018EE2", "F011018EE3"); } else { @upgrade_order = ("F011018EE1", "F011018EE2", "F011018EE3"); } } # If requirements for TSE T030518EE are met already, no upgrade plan is provided. if ($RPS{'T030518EE'}) { my @results = grep {$_ =~ /Drive Support|OneFS/} @upgrade_order; if (!@results and compareVersions($CLUSTER_INFO{'OneFS'}, "8.0.0.6") >= 0) { @upgrade_order = (); } } # If requirements for FCO F033018EE are met already, no upgrade plan is provided. if ($RPS{'F033018EE'}) { my @results = grep {$_ =~ /Drive Support|OneFS/} @upgrade_order; if (!@results and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.2.0") >= 0) { @upgrade_order = (); } } # If requirements for FCO F082318EE are met already, no upgrade plan is provided. if ($RPS{'F082318EE'}) { my @results = grep {$_ =~ /Node Firmware|OneFS/} @upgrade_order; if (!@results and compareVersions($CLUSTER_INFO{'OneFS'}, "8.1.0.4") >= 0) { @upgrade_order = (); } } return @upgrade_order; } # End upgradePlanOrder ### # upgradePlanPrint: Prints out the upgrade plan sub upgradePlanPrint { return unless @_; print "\n"; print colored("\nThe following upgrade plan is for scheduling activities to be performed by Remote Proactive:", 'bold underline'); print "\n"; printSystemDetails(); print "\n"; print "\nSchedule the following activities:"; my $activity_number = 1; if ($RPS{'Issues'} and $RPS{'AutoTC'}) { print "\n$activity_number: Triage Event"; print "\n"; $activity_number++; } foreach my $act_href (@_) { my %activity = %{$act_href}; next unless $activity{'Type'}; my @activities = getActivitiesName($activity{'Type'}, $activity{'Count'}); foreach my $activity_name (@activities) { print "\n$activity_number: $activity_name"; print "\nCurrent Code: $activity{'Current Code'}" if $activity{'Current Code'}; print "\nUpgrading To: $activity{'Target'}" if $activity{'Target'}; print "\nPatch: ".join(", ", @{$activity{'Patches'}}) if $activity{'Patches'}; if ($activity{'Additional Comments'}) { if (scalar(@{$activity{'Additional Comments'}}) == 1) { print "\nAdditional Comments: $activity{'Additional Comments'}[0]"; } else { print "\nAdditional Comments: \n# ".join("\n# ", @{$activity{'Additional Comments'}}); } } print "\n"; $activity_number++; } } } # End upgradePlanPrint exit main();