#!/usr/bin/perl # # IOCA - Isilon On-Cluster Analysis # Current version: 0.1389 # # Copyright (c) 2016-21 Dell Inc. or its subsidiaries. All Rights Reserved. # package IOCA; use Cwd; use Term::ANSIColor; use Time::Local; import Logger qw( :ALL ); import Helper qw( :ALL ); import Parsers qw( :ALL ); import Converters qw( :ALL ); ############################################################## # Global variables # Container for script and upgrade specific information our %RPS = (); @{$RPS{'Upgrade Plan'}} = (); $RPS{'Release Timestamp'} = 1642634221; # date +%s $RPS{'Version'} = '0.1389'; $RPS{'Issues'} = 0; $RPS{'DSP'} = '1.39.1'; $RPS{'IsiFw'} = '11.2'; $RPS{'CheckList'} = 'healthcheck'; # Container for a description of each check our %CHECKS = (); # Container for per node information our %NODE_INFO = (); # Container for cluster wide information our %CLUSTER_INFO = (); # Sorted list of lnns our @LNNS = (); # Conversion table used to lookup lnn from a devid our %LNN_FROM_DEVID = (); # SUDO variable is used for compliance mode clusters our $SUDO = ''; # Set list print behavior $" = $, = ', '; sub main { # Initialize the script initIOCA(); # Run all checks in list &{$_}() for @{$CheckLists{$RPS{'CheckList'}}}; # Print JSON if ($RPS{'JSON'}) { if (@Logger::RESULTS) { print toJSON(\@Logger::RESULTS)."\n"; } return 0; } # If Issues is still 0, then no issues detected if ($RPS{'CheckList'} eq 'pre-upgrade') { if ($RPS{'Issues'} == 0) { print "\nUPGRADE MAY PROCEED\n\n"; } else { print "\nUPGRADE ISSUE DETECTED\n\n"; } } # Display a summary of checks with warnings/failures. # printSummary(); # Develop the upgrade plan upgradePlan() if $RPS{'UpgradePlan'}; return 0; } # Hash of Arrays of node firmware for each device BEGIN { our %CheckLists = ( 'pre-upgrade' => [ 'checkOneFSVersions', 'checkKB544854', 'checkNodeCompatibility', 'checkSupportability', 'checkPatches', 'checkUpgradePath', 'checkCapacity', 'checkHealth', 'checkEvents', 'checkJobStatus', 'checkPartitions', 'checkServices', 'checkServicesMonitoring', 'checkProcesses', 'checkIBInterfaces', 'checkUptime', 'checkUpgrade', 'checkNetworkParallelUpgrade', 'checkHardwareStatus', 'checkBMCandCMC', 'checkBootDisks', 'checkVaultCard', 'checkMirrors', 'checkMemory', 'checkDrivesHealth', 'checkDriveFirmware', 'checkNodeFirmware', 'checkFirmwarePackages', 'checkSmartConnect', 'checkMaintenanceMode', 'checkGatewayPriority', 'checkGroups', 'checkNetBIOS', 'checkSyncIQ', 'checkNDMP', 'checkHDFS', 'checkAggregationMode', 'checkLACPSFP', 'checkDiskpools', 'checkAuthStatus', 'checkZoneLocalAuth', 'checkNFS', 'checkLicense', 'checkISCSI', 'checkAccessZones', 'checkAspera', 'checkContact', 'checkEncoding', 'checkCloudPools', 'checkFilepoolPolicies', 'checkLegacyLDAP', 'checkTimeZone', 'checkLinMasterPadding', 'checkSwitchCompatibility', 'checkUIDGID', # 'checkHTTPConfig', 'checkKB000192800', 'checkKB544401', 'checkKB540872', 'checkKB527312', 'checkKB519119', 'checkKB507031', 'checkKB521890', 'checkKB524082', 'checkKB516613', 'checkKB530050', 'checkKB496993', 'checkKB519423', 'checkKB537785', 'checkKB540000', 'checkKB540071', 'checkKB540513', 'checkKB462202', 'checkKB540901', 'checkKB535582', 'checkDTA000194434', 'checkSSHDConfig', 'checkKerberos8000', 'checkLastZoneID8000', 'checkSBR8000', 'checkNFS8004', 'checkSRS', 'checkKB496582', 'checkKB519388', 'checkKB519890', 'checkETAs', 'checkBXENodes', ], 'firmware' => [ 'checkDriveFirmware', 'checkNodeFirmware', 'checkFirmwarePackages', ], 'healthcheck' => [ 'checkContact', 'checkEmailSettings', 'checkDriveFirmware', 'checkFirmwarePackages', 'checkSupportability', 'checkETAs', 'checkAggregationMode', 'checkBMCandCMC', 'checkBootDisks', 'checkVaultCard', 'checkBXENodes', 'checkDrivesHealth', 'checkDriveLoad', 'checkDriveStallTimeout', 'checkGatewayPriority', 'checkHardwareStatus', 'checkIBInterfaces', 'checkLACPSFP', 'checkMemory', 'checkCloudPools', 'checkMirrors', 'checkNodeCompatibility', 'checkNodeFirmware', 'checkAccessZones', 'checkDiskpools', 'checkFilepoolPolicies', 'checkKerberos8000', 'checkLastZoneID8000', 'checkLegacyLDAP', 'checkNFS8004', 'checkLogLevel', 'checkOneFSVersions', 'checkSBR8000', # 'checkHTTPConfig', 'checkKB544401', 'checkKB540872', 'checkKB507031', 'checkKB519119', 'checkKB521890', 'checkKB530050', 'checkKB516613', 'checkKB496993', 'checkKB540901', 'checkKB519423', 'checkKB537785', 'checkKB535582', 'checkMaintenanceMode', 'checkKB540000', 'checkKB540071', 'checkKB462202', 'checkKB501267', 'checkKB544854', 'checkSSHDConfig', 'checkUpgradePath', 'checkAuthStatus', 'checkCapacity', 'checkEncoding', 'checkSRS', 'checkEvents', 'checkFileSharing', 'checkSwitchCompatibility', 'checkHDFS', 'checkSPNs', 'checkHealth', 'checkIDI', 'checkISCSI', 'checkJobStatus', 'checkJobHistory', 'checkJobs', # 'checkLinMasterPadding', 'checkKB496582', 'checkKB519890', 'checkLicense', 'checkGroups', 'checkLWIODLog', # 'checkMessagesLog', # 'checkNetstat', 'checkListenQueue', 'checkNFS', 'checkOpenFiles', 'checkPartitions', 'checkPatches', 'checkStoragePools', 'checkProcesses', # 'checkRoutingTables', 'checkServices', 'checkServicesMonitoring', 'checkSmartConnect', 'checkSnapshot', 'checkSyncIQ', 'checkTimeDrift', 'checkTimeSync', 'checkTimeZone', 'checkUpgrade', 'checkUptime', ], ); } BEGIN { our %DriveSupport = ( 'None' => undef, '1.39.1' => { 'Dell Ent NVMe AGN RI U.2 1.92TB' => {'2.1.2' => ['allow'],}, 'Dell Ent NVMe AGN RI U.2 15.36TB' => {'2.1.2' => ['allow'],}, 'Dell Ent NVMe AGN RI U.2 3.84TB' => {'2.1.2' => ['allow'],}, 'Dell Ent NVMe AGN RI U.2 7.68TB' => {'2.1.2' => ['allow'],}, 'Dell Ent NVMe FIPS CM6 RI 1.92TB' => {'3.0.6' => ['allow'],}, 'Dell Ent NVMe FIPS CM6 RI 3.84TB' => {'3.0.6' => ['allow'],}, 'Dell Ent NVMe FIPS CM6 RI 7.68TB' => {'3.0.6' => ['allow'],}, 'Dell Ent NVMe FIPS CM6 RI15.36TB' => {'3.0.6' => ['allow'],}, 'Dell Ent NVMe v2 AGN RI U.2 1.92TB' => {'2.0.0' => ['allow'],}, 'Dell Ent NVMe v2 AGN RI U.2 15.36TB' => {'2.0.0' => ['allow'],}, 'Dell Ent NVMe v2 AGN RI U.2 3.84TB' => {'2.0.0' => ['allow'],}, 'Dell Ent NVMe v2 AGN RI U.2 7.68TB' => {'2.0.0' => ['allow'],}, 'HITACHI H54726T6CLAR6000' => {'D9C0' => ['D430'],}, 'HITACHI H54728T8CLAR8000' => {'D9C0' => ['D430'],}, 'HUS726T4TALA6L0' => {'VLEIT9C0' => ['VLEIT430'],}, 'HUS726T6TALN6L0' => {'VKEIT9C0' => ['VKEIT430'],}, }, '1.38.1' => { 'Dell Ent NVMe CM6 RI 1.92TB' => {'2.1.5' => ['allow'],}, 'Dell Ent NVMe CM6 RI 15.36TB' => {'2.1.5' => ['allow'],}, 'Dell Ent NVMe CM6 RI 3.84TB' => {'2.1.5' => ['allow'],}, 'Dell Ent NVMe CM6 RI 7.68TB' => {'2.1.5' => ['allow'],}, 'Dell Express Flash CD5 7.68T SFF' => {'1.2.0' => ['allow'],}, 'HITACHI H54726T4CLAR4000' => {'D9C0' => ['D430'],}, 'KIOXIA KPM6WRUG1T92' => {'BD08' => ['allow'],}, 'KIOXIA KPM6WRUG3T84' => {'BD08' => ['allow'],}, 'KIOXIA KPM6WRUG7T68' => {'BD08' => ['allow'],}, 'KIOXIA KPM6WVUG960G' => {'BD08' => ['allow'],}, 'KIOXIA KPM6XRUG1T92' => {'BA08' => ['allow'],}, 'KIOXIA KPM6XRUG3T84' => {'BA08' => ['allow'],}, 'KIOXIA KPM6XRUG7T68' => {'BA08' => ['allow'],}, 'KIOXIA KPM6XRUG960G' => {'BA08' => ['allow'],}, }, '1.37.2' => { 'Dell Ent NVMe AGN RI U.2 15.36TB' => {'2.0.2' => ['allow'],}, 'Dell Express Flash CD5 7.68T SFF' => {'1.1.5' => ['allow'],}, 'SAMSUNG MZILT7T6HALA0D3' => {'DSA3' => ['allow'],}, 'SEAGATE STCRSKF5CLAR2000' => {'CV06' => ['CV05'],}, 'TOSHIBA KPM5WVUG1T92' => {'B320' => ['allow'],}, 'TOSHIBA KPM5WVUG3T84' => {'B320' => ['allow'],}, 'TOSHIBA KPM5WVUG960G' => {'B320' => ['allow'],}, 'TOSHIBA KPM5XRUG7T68' => {'B01C' => ['allow'],}, }, '1.35' => { 'Dell Ent NVMe AGN RI U.2 1.92TB' => {'2.0.2' => ['allow'],}, 'Dell Ent NVMe AGN RI U.2 3.84TB' => {'2.0.2' => ['allow'],}, 'Dell Ent NVMe AGN RI U.2 7.68TB' => {'2.0.2' => ['allow'],}, 'Dell Express Flash CD5 7.68T SFF' => {'1.1.1' => ['allow'],}, 'Dell Express Flash NVMe P4510 2TB SFF' => {'VDV1DP23' => ['allow'],}, 'Dell Express Flash NVMe P4510 4TB SFF' => {'VDV1DP23' => ['allow'],}, 'SAMSUNG MZILT1T9HBJR0D3' => {'DSA3' => ['allow'],}, 'SAMSUNG MZILT3T8HBLS0D3' => {'DSA3' => ['allow'],}, 'SAMSUNG MZILT960HBHQ0D3' => {'DSA3' => ['allow'],}, 'TOSHIBA KPM5XRUG1T92' => {'B01C' => ['allow'],}, 'TOSHIBA KPM5XRUG3T84' => {'B01C' => ['allow'],}, 'TOSHIBA KPM5XRUG960G' => {'B01C' => ['allow'],}, 'TOSHIBA KRM5XRUG1T92' => {'B709' => ['allow'],}, 'TOSHIBA KRM5XRUG3T84' => {'B709' => ['allow'],}, 'TOSHIBA KRM5XRUG960G' => {'B709' => ['allow'],}, }, '1.34' => undef, '1.33' => { 'HGST HUC101812CSS200' => {'AE00' => ['A9C1', 'AD40'],}, 'HGST HUC101830CSS200' => {'AE00' => ['AAA2', 'AD40'],}, 'HGST HUC101860CSS200' => {'AE00' => ['AAA2', 'AD40'],}, 'HGST HUC101890CSS200' => {'AE00' => ['AAA2', 'AD40'],}, 'HGST HUC101890CSS205' => {'DE00' => ['DAA2', 'DDY0'],}, 'HITACHI H4SMM321CLAR1600' => {'S177' => ['S116'],}, 'HITACHI H4SMM328 CLAR800' => {'S177' => ['S116'],}, 'HITACHI H4SMR321CLAR1600' => {'S177' => ['S116'],}, 'HITACHI H4SMR323CLAR3200' => {'S177' => ['S116'],}, 'HITACHI H4SMR324 CLAR400' => {'S177' => ['S116'],}, 'HITACHI H4SMR328 CLAR800' => {'S177' => ['S116'],}, 'HITACHI H4STR763CLAR3840' => {'S52K' => ['S520'],}, 'HITACHI H4STR767CLAR7680' => {'S52K' => ['S520'],}, 'HITACHI H5C10181CLAR1200' => {'DE00' => ['DAA2', 'DDY0'],}, 'HITACHI H5C10186 CLAR600' => {'DE00' => ['DAA2', 'DDY0'],}, 'HITACHI H5H72101CLAR10T0' => {'D614' => ['D32A'],}, 'HITACHI H5H72108CLAR8000' => {'D614' => ['D32A'],}, 'HITACHI H5SMM321CLAR1600' => {'D176' => ['S116'],}, 'HITACHI H5SMM328 CLAR800' => {'D176' => ['S116'],}, 'HITACHI H5SMR321CLAR1600' => {'D176' => ['S116'],}, 'HITACHI H5SMR323CLAR3200' => {'D176' => ['S116'],}, 'HITACHI H5SMR324 CLAR400' => {'D176' => ['S116'],}, 'HITACHI H5SMR328 CLAR800' => {'D176' => ['S116'],}, 'HITACHI HUC10181CLAR1200' => {'AE00' => ['L9H0'],}, 'HITACHI HUC10186 CLAR600' => {'AE00' => ['L9H0'],}, 'HUS726020ALN610' => {'APGNTDY0' => ['APGNT840', 'APGNTDCH'],}, 'HUS726040ALN610' => {'APGNTDY0' => ['APGNT840', 'APGNTDCH'],}, 'SEAGATE STMFSKF2CLAR4000' => {'BV01' => ['BV00'],}, 'SEAGATE STMFSKF4CLAR2000' => {'BV01' => ['BV00'],}, }, '1.32' => { 'HUS726020ALA610' => {'A5EITDY0' => ['A5EIT922', 'A5EITD40'],}, 'HUS726040ALA610' => {'A5EITDY0' => ['A5EIT922', 'A5EITD40'],}, 'HUS726060ALN610' => {'APEITDY0' => ['APEIT922', 'APEITD40'],}, }, '1.31' => undef, '1.30' => { 'HDS725050KLA360' => {'K2AOAD1A' => ['K2AOA10C', 'K2AOA51A', 'K2AOAB0A'],}, 'HGST H5472606CLAR6000' => {'NDY0' => ['N9L0'],}, 'HGST H5H72808CLAR8000' => {'MDY0' => ['M9L0'],}, 'HGST HUC101812CSS200' => {'AD40' => ['A9C1'],}, 'HGST HUC101830CSS200' => {'AD40' => ['AAA2'],}, 'HGST HUC101860CSS200' => {'AD40' => ['AAA2'],}, 'HGST HUC101890CSS200' => {'AD40' => ['AAA2'],}, 'HGST HUC101890CSS205' => {'DDY0' => ['DAA2'],}, 'HGST HUS724020ALA640' => {'MF6OAC50' => ['MF6OA8B0', 'MF6OAAC0', 'MF6OABW0'],}, 'HGST HUS724030ALA640' => {'MF8OAC50' => ['MF8OA8B0', 'MF8OAAC0', 'MF8OABW0'],}, 'HGST HUS724040ALA640' => {'MFAOAC50' => ['MFAOA8B0', 'MFAOAAC0', 'MFAOABW0'],}, 'HGST HUS726060ALA640' => {'AHIST1EZ' => ['AHIST1ER'],}, 'HGST HUSMM1620ASS200' => {'A204' => ['A100'],}, 'HGST HUSMM1640ASS200' => {'A204' => ['A100'],}, 'HGST HUSMM1680ASS200' => {'A204' => ['A100'],}, 'HGST HUSMM1680ASS205' => {'D32A' => ['D252', 'd322', 'D326', 'D328'],}, 'HGST HUSMM8020ASS200' => {'A210' => ['A100', 'A160'],}, 'HGST HUSMM8040ASS200' => {'A210' => ['A100', 'A160'],}, 'HGST HUSMM8080ASS200' => {'A210' => ['A100', 'A160'],}, 'HITACHI H5472602CLAR2000' => {'DDW0' => ['D930', 'DD9H'],}, 'HITACHI H5472604CLAR4000' => {'DDW0' => ['D930', 'DD9H'],}, 'HITACHI H5C10181CLAR1200' => {'DDY0' => ['DAA2'],}, 'HITACHI H5C10186 CLAR600' => {'DDY0' => ['DAA2'],}, 'HITACHI HUC106030CSS600' => {'A430' => ['A180', 'A202'],}, 'HITACHI HUC106060CSS600' => {'A430' => ['A180', 'A202'],}, 'HITACHI HUC109030CSS600' => {'A5B0' => ['A222', 'A2D0', 'A350', 'A440'],}, 'HITACHI HUC109060CSS600' => {'A5B0' => ['A222', 'A2D0', 'A350', 'A440'],}, 'HITACHI HUC109090CSS600' => {'A5B0' => ['A222', 'A2D0', 'A350', 'A440'],}, 'HITACHI HUS156045VLS600' => {'A760' => ['A5D0'],}, 'HITACHI HUSMM151CLAR1600' => {'D348' => ['D344', 'D346'],}, 'HITACHI HUSSL4010ASS600' => {'A182' => ['A170'],'A18M' => ['A131'],}, 'HITACHI HUSSL4010BSS600' => {'A1D0' => ['A090', 'A110', 'A190'],}, 'HITACHI HUSSL4020ASS600' => {'A182' => ['A170'],'A18M' => ['A131'],}, 'HITACHI HUSSL4020BSS600' => {'A1D0' => ['A090', 'A110', 'A190'],}, 'HITACHI HUSSL4040ASS600' => {'A182' => ['A170'],'A18M' => ['A131'],}, 'HITACHI HUSSL4040BSS600' => {'A1D0' => ['A090', 'A110', 'A190'],}, 'HUH728080ALN600' => {'A4EITD40' => ['A4EIT921'],}, 'HUS726020ALA610' => {'A5EITD40' => ['A5EIT922'],}, 'HUS726020ALN610' => {'APGNTDCH' => ['APGNT840'],}, 'HUS726040ALA610' => {'A5EITD40' => ['A5EIT922'],}, 'HUS726040ALN610' => {'APGNTDCH' => ['APGNT840'],}, 'HUS726060ALN610' => {'APEITD40' => ['APEIT922'],}, 'Hitachi HUA722010CLA330' => {'JP4OA3PA' => ['JP4OA3GB', 'JP4OA3MA'],}, 'Hitachi HUA722020ALA330' => {'JKAOA3PA' => ['JKAOA3GA', 'JKAOA3MA', 'JKAOA20N', 'JKAOA28A'],}, 'Hitachi HUA722050CLA330' => {'JP2OA3PA' => ['JP2OA3GB', 'JP2OA3MA'],}, 'Hitachi HUA723020ALA640' => {'MK7OAA50' => ['MK7OA6L0', 'MK7OA8P0', 'MK7OA580', 'MK7OA5V0'],}, 'Hitachi HUA723030ALA640' => {'MKAOAA50' => ['MKAOA6L0', 'MKAOA8P0', 'MKAOA3B0', 'MKAOA580', 'MKAOA5V0'],}, 'SAMSUNG P043S15T EMC15T3' => {'ESV5' => ['ESV4'],}, 'SEAGATE ST3000NM0063' => {'G007' => ['0003', '0004', '0006', 'C007', 'F007'],}, 'SEAGATE ST33000652SS' => {'A006' => ['0005', '0006'],}, 'SEAGATE ST4000NM0063' => {'G007' => ['0003', '0004', '0006', 'C007', 'F007'],}, 'SEAGATE ST6000NM0114' => {'KFH5' => ['KF05'],}, 'ST1000NM0033-9ZM173' => {'EN09' => ['SN03', 'SNG4', 'EN06', 'EN07'],}, 'ST2000NM0033-9ZM175' => {'EN09' => ['SN03', 'SNG4', 'EN06', 'EN07'],}, 'ST3000NM0033-9ZM178' => {'EN09' => ['SN03', 'SNG4', 'EN06', 'EN07'],}, 'ST4000NM0033-9ZM170' => {'EN09' => ['SN02', 'SN03', 'SNG4', 'EN06', 'EN07'],}, 'ST8000NM0045-1RL112' => {'UG07' => ['UG05'],}, 'STEC M8ISB2B-200UC' => {'2832' => ['2686'],}, 'STEC MACH8 IOPS' => {'2828' => ['2269', '2366', '2608', '2755'],}, 'WDC WD1002FBYS-02A6B0' => {'03.02C06' => ['03.01C06'],}, }, ); } BEGIN { our @NFP = ('11.2', '11.1.4', '10.3.6', '11.1', '10.3.5', '11.0', '10.3.3', '10.3.2', '10.3.1', '10.3.0', '9.3.5'); our %NodeFirmware = ( 'None' => undef, '11.2' => { 'BIOS' => ['2.8.2'], 'BMCSDR_A100' => ['01.07'], 'BP14G+ 0:1' => ['4.35'], 'BP14G+EXP 0:1' => ['2.46'], 'Backplane 1' => ['2.46', '4.35'], 'Backplane 2' => ['2.46', '4.35'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.29'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.4+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.29'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.14'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0003.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'Dell HBA330 Mini' => ['16.17.01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.37'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.4+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.37'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.13'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0003.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'Integrated Remote Access Controller' => ['4.40.10.00'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'MLKadpt_skylanders' => ['02.80'], 'MLKbem_mongoose' => ['02.20'], 'MLKbios_gryphon' => ['09.24'], 'MLKccdes' => ['1.10.4+21.00'], 'MLKcdes' => ['2.41.1+0.18.0+21.00'], 'MLKcec_region0' => ['02.00'], 'MLKcec_region1' => ['32.02'], 'MLKcec_region3' => ['00.03'], 'MLKcmd_100dc12pin' => ['02.02.72.03'], 'MLKcmd_64pin' => ['02.05.102.02'], 'MLKcmd_rev_table' => ['20.66'], 'MLKeth_infinitymlk' => ['02.64'], 'MLKfp_fpinffw' => ['02.0c'], 'MLKfp_fpioc' => ['00.0b'], 'MLKfullbios_gryphon_post' => ['09.24'], 'MLKfw_map_table' => ['40.49'], 'MLKgem_skylanders' => ['37.31'], 'MLKpost_gryphon' => ['09.36'], 'MLKps0_gen2_artesyn_optimus' => ['02.14'], 'MLKps0_gen2_artesyn_ramjet' => ['02.14'], 'MLKps1_gen2_artesyn_optimus' => ['02.14'], 'MLKps1_gen2_artesyn_ramjet' => ['02.14'], 'MLKsas_catamaran' => ['0015.0004.0000.0000'], 'MLKsas_infinitymlk' => ['0001.0004.0000.11181'], 'MLKserdes_infinitymlk' => ['00.30'], 'MLKspime_gryphon' => ['04.00'], 'MLKssp_infinitymlk' => ['01.10'], 'MLKuefi_fw_volume' => ['40.79'], 'MLKvrd0_infinitymlk_infineon' => ['01.01'], 'MLKvrd1_infinitymlk_infineon' => ['01.01'], 'MLKvrd2_infinitymlk_infineon' => ['01.01'], 'MLKvrd3_infinitymlk_infineon' => ['01.02'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-CX4' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4-BE' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4R' => ['14.27.6008+DEL2810000034'], 'Mellanox-CX5-BE' => ['16.27.6106+DEL0000000004'], 'Mellanox-CX5-FE0' => ['16.27.6106+DEL0000000004'], 'Mellanox-CX6-BE' => ['22.28.4512+DEL0000000027'], 'Mellanox-CX6-FE0' => ['22.28.4512+DEL0000000027'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.28.1002+EMC0000000017'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], }, '11.1.4' => { 'BIOS' => ['2.8.2'], 'BMCSDR_A100' => ['01.07'], 'BP14G+ 0:1' => ['4.35'], 'BP14G+EXP 0:1' => ['2.46'], 'Backplane 1' => ['2.46', '4.35'], 'Backplane 2' => ['2.46', '4.35'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.29'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.4+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.29'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.14'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0003.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'Dell HBA330 Mini' => ['16.17.01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.37'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.4+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.37'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.13'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0003.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'Integrated Remote Access Controller' => ['4.40.10.00'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-CX4' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4-BE' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4R' => ['14.27.6008+DEL2810000034'], 'Mellanox-CX5-BE' => ['16.27.6106+DEL0000000004'], 'Mellanox-CX5-FE0' => ['16.27.6106+DEL0000000004'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.28.1002+EMC0000000017'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], }, '11.1' => { 'BIOS' => ['2.8.2'], 'BMCSDR_A100' => ['01.07'], 'BP14G+ 0:1' => ['4.35'], 'BP14G+EXP 0:1' => ['2.46'], 'Backplane 1' => ['2.46', '4.35'], 'Backplane 2' => ['2.46', '4.35'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.28'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.2+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.28'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.09'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'Dell HBA330 Mini' => ['16.17.01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.35'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.2+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.35'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.09'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'Integrated Remote Access Controller' => ['4.20.20.20'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-CX4' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4-BE' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4R' => ['14.27.6008+DEL2810000034'], 'Mellanox-CX5-BE' => ['16.27.6106+DEL0000000004'], 'Mellanox-CX5-FE0' => ['16.27.6106+DEL0000000004'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], }, '11.0' => { 'BIOS' => ['2.4.8'], 'BMCSDR_A100' => ['01.07'], 'BP14G+ 0:1' => ['4.35'], 'BP14G+EXP 0:1' => ['2.46'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.24'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.2+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.24'], 'DEfw_map_table' => ['12.14'], 'DEpost_banshee' => ['28.07'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'Dell HBA330 Mini' => ['16.17.00.05'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.32'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.2+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.32'], 'EPfw_map_table' => ['12.14'], 'EPpost_warthog' => ['22.07'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'Integrated Remote Access Controller' => ['4.00.00.00'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-CX4' => ['14.25.8000+DEL2420110034'], 'Mellanox-CX4-BE' => ['14.25.8000+DEL2420110034'], 'Mellanox-CX4R' => ['14.25.8000+DEL2810000034'], 'Mellanox-CX5-BE' => ['16.25.8000+DEL0000000004'], 'Mellanox-CX5-FE0' => ['16.25.8000+DEL0000000004'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], }, '10.3.6' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.29'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.4+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.29'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.14'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0003.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.37'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.4+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.37'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.13'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0003.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['14.25.1020+DEL2420110034', '16.23.4010+MT_0000000158', '16.28.1002+EMC0000000017'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }, '10.3.5' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.28'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.2+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.28'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.09'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.35'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.2+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.35'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.09'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '14.25.1020+DEL2420110034', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }, '10.3.3' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.24'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.20'], 'DEccdes' => ['1.10.2+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.24'], 'DEfw_map_table' => ['12.14'], 'DEpost_banshee' => ['28.07'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.81'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.32'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.20'], 'EPccdes' => ['1.10.2+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.32'], 'EPfw_map_table' => ['12.14'], 'EPpost_warthog' => ['22.07'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.81'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '14.25.1020+DEL2420110034', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }, '10.3.2' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.22'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.00'], 'DEccdes' => ['1.10.1+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.22'], 'DEfw_map_table' => ['12.14'], 'DEpost_banshee' => ['28.07'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.81'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.31'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.00'], 'EPccdes' => ['1.10.1+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.31'], 'EPfw_map_table' => ['12.14'], 'EPpost_warthog' => ['22.07'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.81'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }, '10.3.1' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.22'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.00'], 'DEccdes' => ['1.10.1+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.22'], 'DEfw_map_table' => ['12.14'], 'DEpost_banshee' => ['28.07'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.80'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.31'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.00'], 'EPccdes' => ['1.10.1+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.31'], 'EPfw_map_table' => ['12.14'], 'EPpost_warthog' => ['22.07'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.80'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }, '10.3.0' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['01.20'], 'DEbios_banshee' => ['37.19'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.00'], 'DEccdes' => ['1.10.1+21.00'], 'DEcdes' => ['2.37.1+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEefi_sas_catamaran' => ['0017.0000.0003.0000'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.19'], 'DEfw_map_table' => ['12.14'], 'DEpost_banshee' => ['28.05'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0002.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11181'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.01'], 'DEssp_infinity' => ['02.80'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['01.20'], 'EPbios_warthog' => ['32.27'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.00'], 'EPccdes' => ['1.10.1+21.00'], 'EPcdes' => ['2.37.1+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPefi_sas_catamaran' => ['0017.0000.0003.0000'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.27'], 'EPfw_map_table' => ['12.14'], 'EPpost_warthog' => ['22.05'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0002.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11181'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['07.00'], 'EPssp_infinity' => ['02.80'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1004+0x1004', '0x1002+0x1002'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.24.4020+EMC0000000017', '16.23.4010+MT_0000000158'], 'PMC8056' => ['2.13.0+0.11.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }, '9.3.5' => { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.05'], 'CAR_Ermine' => ['02.05'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.03', '02.02', '09.05', '06.06'], 'CFFPS1_Blastoff' => ['03.03'], 'CFFPS1_Octane' => ['09.05'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.03', '02.02', '09.05', '06.06'], 'CFFPS2_Blastoff' => ['03.03'], 'CFFPS2_Octane' => ['09.05'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.05'], 'CMC_Ermine' => ['02.05'], 'CMC_HFHB' => ['02.05'], 'CMC_Yeti' => ['02.05'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180c01', 'rp180c01+rp180c01'], 'LOx2' => ['rp180c01', 'rp180c01+rp180c01'], 'LOx2-MLC-YD' => ['rp180c01', 'rp180c01+rp180c01'], 'Lsi' => ['1.023', '6.28.00.00+01.28.02.00+1.17', '0.99c', '6.28.00.00+01.28.02.00+1.00', '13.00.66.00', '20.00.04.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['2.30.8020+ISL1090110018', '2.30.8020+MT_1090110018', '4.8.930+MT_02F0140002'], 'Mellanox-EN' => ['2.34.5000+MT_1090111023'], 'PMC8056' => ['2.13.0+0.11.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'Vitesse0' => ['1.024', '0.99c'], 'Vitesse1' => ['1.024'], }, ); } # Supported out-of-family upgrade paths # Source => Target versions # Highest Sources at the top # Target versions are the lowest supported version that can be upgraded to at each family excluding within family # Target versions MUST be 4 decimal separated digits in length # Target versions are generated based on the keys when no target is found in the list BEGIN { our %Code_Matrix = ( '9.3.0' => [ ], '9.2.1' => [ '9.3.0.0', ], '9.2.0' => [ '9.3.0.0', '9.2.1.0', ], '9.1.1' => [ '9.3.0.0' ], '9.1.0' => [ '9.3.0.0', '9.2.1.0', '9.2.0.0' ], '9.0.0.0' => [ '9.3.0.0', '9.2.1.0', '9.2.0.0', '9.1.0.0', ], '8.2.2.0' => [ '9.3.0.0', '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', ], '8.2.1.0' => [ '9.3.0.0', '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', ], '8.2.0.0' => [ '9.3.0.0', '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', ], '8.1.3.0' => [ '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', ], '8.1.2.0' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0' ], '8.1.1.1' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0' ], '8.1.1.0' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0' ], '8.1.0.4' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0' ], '8.1.0.3' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0' ], '8.1.0.2' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0' ], '8.1.0.1' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0' ], '8.1.0.0' => [ '9.2.1.0', '9.2.0.0', '9.1.0.0', '9.0.0.0', '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0' ], '8.0.1.2' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.2' ], '8.0.1.1' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.1' ], '8.0.1.0' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0' ], '8.0.0.7' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0' ], '8.0.0.6' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.2' ], '8.0.0.5' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.1', '8.0.1.2' ], '8.0.0.4' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.1' ], '8.0.0.3' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.1' ], '8.0.0.2' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.1' ], '8.0.0.1' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0' ], '8.0.0.0' => [ '8.2.2.0', '8.2.1.0', '8.2.0.0', '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0' ], '7.2.1.6' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.1', '8.0.1.2', '8.0.0.5' ], '7.2.1.5' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.1', '8.0.0.4' ], '7.2.1.4' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.1', '8.0.0.3' ], '7.2.1.3' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.2' ], '7.2.1.2' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.1' ], '7.2.1.1' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.0' ], '7.2.1.0' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.0' ], '7.2.0.5' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.1', '7.2.1.2' ], '7.2.0.4' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.0', '7.2.1.1' ], '7.2.0.3' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.0', '7.2.1.1' ], '7.2.0.2' => [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.1.0.0', '8.0.1.0', '8.0.0.0', '7.2.1.0' ], '7.2.0.1' => [ ], '7.2.0.0' => [ ], '7.1.1.11'=> [ '8.1.3.0', '8.1.2.0', '8.1.1.0', '8.0.1.1', '8.0.0.4', '7.2.1.5' ], '7.1.1.9' => [ '8.0.1.0', '8.0.0.1', '7.2.1.3' ], '7.1.1.8' => [ '8.0.1.0', '8.0.0.1', '7.2.1.2' ], '7.1.1.7' => [ '8.0.1.0', '8.0.0.0', '7.2.1.1', '7.2.0.4' ], '7.1.1.6' => [ '8.0.1.0', '8.0.0.0', '7.2.1.1', '7.2.0.3' ], '7.1.1.5' => [ '8.0.1.0', '8.0.0.0', '7.2.1.1', '7.2.0.3' ], '7.1.1.4' => [ '8.0.1.0', '8.0.0.0', '7.2.1.0', '7.2.0.2' ], '7.1.1.3' => [ '7.2.0.1' ], '7.1.1.2' => [ '7.2.0.1' ], '7.1.1.1' => [ '7.2.0.0' ], '7.1.1.0' => [ '7.2.0.0' ], '7.1.0' => [ '7.2.0.1', '7.1.1.2' ], '7.0' => [ '7.2.0.2', '7.1.1.3' ], '6.5.5.29'=> [ '7.1.1.1', '7.1.0.0' ], '6.5' => [ '7.1.1.1', '7.1.0.0' ], '6.0' => [ '6.5.5.29' ], ); } # Activity names BEGIN { our %Activities = ( 'FCO F042415EE' => { 5 => 'FCO F042415EE Drive Firmware Upgrade 1-5 nodes', 10 => 'FCO F042415EE Drive Firmware Upgrade 6-10 nodes', 15 => 'FCO F042415EE Drive Firmware Upgrade 11-15 nodes', 20 => 'FCO F042415EE Drive Firmware Upgrade 16-20 nodes', 25 => 'FCO F042415EE Drive Firmware Upgrade 21-25 nodes', }, 'Isilon Hourly' => { 1 => 'Isilon 1 Hour', 1.5 => 'Isilon 1.5 Hours', 2 => 'Isilon 2 Hours', 2.5 => 'Isilon 2.5 Hours', 3 => 'Isilon 3 Hours', 3.5 => 'Isilon 3.5 Hours', 4 => 'Isilon 4 Hours', 4.5 => 'Isilon 4.5 Hours', 5 => 'Isilon 5 Hours', 5.5 => 'Isilon 5.5 Hours', 6 => 'Isilon 6 Hours', 6.5 => 'Isilon 6.5 Hours', 7 => 'Isilon 7 Hours', 7.5 => 'Isilon 7.5 Hours', 8 => 'Isilon 8 Hours', 8.5 => 'Isilon 8.5 Hours', 9 => 'Isilon 9 Hours', 9.5 => 'Isilon 9.5 Hours', 10 => 'Isilon 10 Hours', 10.5=> 'Isilon 10.5 Hours', 11 => 'Isilon 11 Hours', 11.5=> 'Isilon 11.5 Hours', 12 => 'Isilon 12 Hours', }, ); } # Code Links BEGIN { my $DL_EOSL = 'EOSL code, not available for direct download. File must be provided via temporary FTP.'; my $DL_DA = 'This code is directed availability. File must be provided via temporary FTP.'; ### # Bundles: With exception of the OneFS installation file, a list of the files # included in each installation bundle. our %Bundles = ( 'Isilon OneFS 9.3.0.3 installation bundle' => [ 'Isilon Node Firmware Package 11.2', 'Isilon Drive Support Package 1.39.1', 'IsiFru Package', ], 'Isilon OneFS 9.3.0.2 installation bundle' => [ 'Isilon Node Firmware Package 11.3', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 9.3.0.1 installation bundle' => [ 'Isilon Node Firmware Package 11.3', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.7 installation bundle' => [ 'Isilon Node Firmware Package 11.3', 'Isilon Drive Support Package 1.39.1', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.6 installation bundle' => [ 'Isilon Node Firmware Package 11.3', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.5 installation bundle' => [ 'Isilon Node Firmware Package 11.1.4', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.4 installation bundle' => [ 'Isilon Node Firmware Package 11.1.4', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.3 installation bundle' => [ 'Isilon Node Firmware Package 11.1.4', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.2 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.2.1.1 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.2.0.1 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.14 installation bundle' => [ 'Isilon Node Firmware Package 11.3', 'Isilon Drive Support Package 1.39.1', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.13 installation bundle' => [ 'Isilon Node Firmware Package 11.3', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.12 installation bundle' => [ 'Isilon Node Firmware Package 11.1.4', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.11 installation bundle' => [ 'Isilon Node Firmware Package 11.1.4', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.10 installation bundle' => [ 'Isilon Node Firmware Package 11.1.4', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.9 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.8 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.7 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.6 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.35', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.5 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.35', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.4 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.35', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.3 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.34', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.2 installation bundle' => [ 'Isilon Node Firmware Package 11.1', 'Isilon Drive Support Package 1.34', 'IsiFru Package', ], 'Isilon OneFS 9.1.0.1 installation bundle' => [ 'Isilon Node Firmware Package 11.0', 'Isilon Drive Support Package 1.34', 'IsiFru Package', ], 'Isilon OneFS 9.0.0.0 installation bundle' => [ 'Isilon Node Firmware Package 11.0', 'Isilon Drive Support Package 1.34', 'IsiFru Package', ], 'Isilon OneFS 8.2.2.0 installation bundle' => [ 'Isilon Node Firmware Package 10.3.7', 'Isilon Drive Support Package 1.38.1', 'IsiFru Package', ], 'Isilon OneFS 8.2.1.0 installation bundle' => [ 'Isilon Node Firmware Package 10.3.5', 'Isilon Drive Support Package 1.35', 'IsiFru Package', ], 'Isilon OneFS 8.2.0.0 installation bundle' => [ 'Isilon Node Firmware Package 10.3.3', 'Isilon Drive Support Package 1.33', 'IsiFru Package', ], 'Isilon OneFS 8.1.2.0 installation bundle' => [ 'Isilon Node Firmware Package 10.3.5', 'Isilon Drive Support Package 1.37.2', 'IsiFru Package', ], ); ### # Patched_OneFS: For pre-patched OneFS images, includes a list of # the patches pre-loaded in the corresponding image. our %Patched_OneFS = ( '9.3.0.3' => [ '9.3.0.3_GA-RUP_2022-01_PSP-1724', ], '9.3.0.2' => [ '9.3.0.2_GA-RUP_2021-12_PSP-1643', ], '9.3.0.1' => [ '9.3.0.1_GA-RUP_2021-11_PSP-1764', ], '9.3.0.0' => undef, '9.2.1.7' => [ '9.2.1.7_GA-RUP_2021-12_PSP-1698', 'HealthCheck_9.2.1_2021-09', ], '9.2.1.6' => [ '9.2.1.6_GA-RUP_2021-11_PSP-1640', 'HealthCheck_9.2.1_2021-09', ], '9.2.1.5' => [ '9.2.1.5_GA-RUP_2021-10_PSP-1525', 'HealthCheck_9.2.1_2021-09', '9.X_GA-TOOLS_2021-06_PSP-1306', ], '9.2.1.4' => [ '9.2.1.4_GA-RUP_2021-09_PSP-1432', 'HealthCheck_9.2.1_2021-09', '9.X_GA-TOOLS_2021-06_PSP-1306', ], '9.2.1.3' => [ '9.2.1.3_GA-RUP_2021-08_PSP-1394', 'HealthCheck_9.2.1_2021-06', '9.X_GA-TOOLS_2021-06_PSP-1306', ], '9.2.1.2' => [ '9.2.1.2_GA-RUP_2021-07_PSP-1246', 'HealthCheck_9.2.1_2021-06', '9.X_GA-TOOLS_2021-06_PSP-1306', ], '9.2.1.1' => [ '9.2.1.1_GA-RUP_2021-06_PSP-1313', 'HealthCheck_9.2.1_2021-06' ], '9.2.1.0' => undef, '9.2.0.1' => [ '9.2.0.1_GA-RUP_2021-06_PSP-1322', 'HealthCheck_9.2.0_2021-06', ], '9.2.0.0' => undef, '9.1.0.14' => ['9.1.0.14_GA-RUP_2021-12_PSP-1722', 'HealthCheck_9.1.0_2021-09', ], '9.1.0.13' => ['9.1.0.13_GA-RUP_2021-11_PSP-1650', 'HealthCheck_9.1.0_2021-09', ], '9.1.0.12' => ['9.1.0.12_GA-RUP_2021-10_PSP-1548', '9.X_GA-TOOLS_2021-06_PSP-1306', 'HealthCheck_9.1.0_2021-09', ], '9.1.0.11' => ['9.1.0.11_GA-RUP_2021-09_PSP-1496', '9.X_GA-TOOLS_2021-06_PSP-1306', 'HealthCheck_9.1.0_2021-09', ], '9.1.0.10' => ['9.1.0.10_GA-RUP_2021-08_PSP-1398', '9.X_GA-TOOLS_2021-06_PSP-1306', 'HealthCheck_9.1.0_2021-06', ], '9.1.0.9' => [ '9.1.0.9_GA-RUP_2021-07_PSP-1248', '9.X_GA-TOOLS_2021-06_PSP-1306', 'HealthCheck_9.1.0_2021-06', ], '9.1.0.8' => [ '9.1.0.8_GA-RUP_2021-06_PSP-1118', '9.X_GA-TOOLS_2021-06_PSP-1306', 'HealthCheck_9.1.0_2021-06', ], '9.1.0.7' => [ '9.1.0.7_GA-RUP_2021-05_PSP-1037', 'MULTI_GA-TOOL_2021-05_PSP-1034', 'HealthCheck_9.1.0_2021-03', ], '9.1.0.6' => [ '9.1.0.6_GA-RUP_2021-04_PSP-928', 'HealthCheck_9.1.0_2021-03', ], '9.1.0.5' => [ '9.1.0.5_GA-RUP_2021-03_PSP-842', 'HealthCheck_9.1.0_2021-03', ], '9.1.0.4' => [ '9.1.0.4_GA-RUP_2021-02_PSP-769', 'HealthCheck_9.1.0_2021-02', ], '9.1.0.3' => [ '9.1.0.3_GA-RUP_2021-01_PSP-664', 'HealthCheck_9.1.0_2021-01', ], '9.1.0.2' => [ '9.1.0.2_GA-RUP_2020-12_PSP-524', 'HealthCheck_9.1.0_2020-12', ], '9.1.0.1' => [ '9.1.0.1_GA-RUP_2020-11_PSP-464', 'HealthCheck_9.1.0_2020-10', ], '9.1.0.0' => undef, '9.0.0.0' => [ '9.0.0_UGA-PATCH-INFRA_2020-07_PSP-53', '9.0.0_GA-RUP_2020-11_PSP-406', 'HealthCheck_9.0.0_2020-11', ], '8.2.2.0' => [ '8.2.2_UGA-PATCH-INFRA_2020-06_274207', '8.2.2_GA-RUP_2021-11_PSP-1571', 'HealthCheck_8.2.2_2021-09', '8.X_GA-TOOLS_2021-05_PSP-1034', ], '8.2.1.0' => [ '8.2.1_GA-RUP_2021-01_PSP-397', '8.2.1_UGA-PATCH-INFRA_2020-03_267223', 'HealthCheck_8.2.1_2021-01', ], '8.2.0.0' => [ '8.2.0_UGA-PATCH-INFRA_2019-11_263084', '8.2.0_GA-RUP_2020-06_273332', 'HealthCheck_8.2.0_2020-06', ], '8.1.2.0' => [ '8.1.2.0_UGA-PATCH-INFRA_2019-12_263531', '8.1.2.0_GA-RUP_2021-06_PSP-1194', '8.X_GA-TOOLS_2021-05_PSP-1034', 'HealthCheck_8.1.2_2021-05', ], ); ### # Patch_Releases: For OneFS 9.1 and later releases. A table of which patches # correspond to each OneFS version. our %Patch_Releases = ( '9.1.0.0' => undef, '9.1.0.1' => { 'GA-RUP' => '9.1.0.1_GA-RUP_2020-11_PSP-464', }, '9.1.0.2' => { 'GA-RUP' => '9.1.0.2_GA-RUP_2020-12_PSP-524', 'DA-RUP' => '9.1.0.2_DA-RUP_2020-12_PSP-645', }, '9.1.0.3' => { 'GA-RUP' => '9.1.0.3_GA-RUP_2021-01_PSP-664', 'DA-RUP' => '9.1.0.3_DA-RUP_2021-01_PSP-665', }, '9.1.0.4' => { 'GA-RUP' => '9.1.0.4_GA-RUP_2021-02_PSP-769', 'DA-RUP' => '9.1.0.4_DA-RUP_2021-02_PSP-770', }, '9.1.0.5' => { 'GA-RUP' => '9.1.0.5_GA-RUP_2021-03_PSP-842', 'DA-RUP' => '9.1.0.5_DA-RUP_2021-03_PSP-843', }, '9.1.0.6' => { 'GA-RUP' => '9.1.0.6_GA-RUP_2021-04_PSP-928', 'DA-RUP' => '9.1.0.6_DA-RUP_2021-04_PSP-929', }, '9.1.0.7' => { 'GA-RUP' => '9.1.0.7_GA-RUP_2021-05_PSP-1037', 'DA-RUP' => '9.1.0.7_DA-RUP_2021-05_PSP-1038', }, '9.1.0.8' => { 'GA-RUP' => '9.1.0.8_GA-RUP_2021-06_PSP-1118', 'DA-RUP' => '9.1.0.8_DA-RUP_2021-06_PSP-1119', }, '9.1.0.9' => { 'GA-RUP' => '9.1.0.9_GA-RUP_2021-07_PSP-1248', 'DA-RUP' => '9.1.0.9_DA-RUP_2021-07_PSP-1249', }, '9.1.0.10' =>{ 'GA-RUP' => '9.1.0.10_GA-RUP_2021-08_PSP-1398', 'DA-RUP' => '9.1.0.10_DA-RUP_2021-08_PSP-1399', }, '9.1.0.11' =>{ 'GA-RUP' => '9.1.0.11_GA-RUP_2021-09_PSP-1496', 'DA-RUP' => '9.1.0.11_DA-RUP_2021-09_PSP-1497', }, '9.1.0.12' =>{ 'GA-RUP' => '9.1.0.12_GA-RUP_2021-10_PSP-1548', 'DA-RUP' => '9.1.0.12_DA-RUP_2021-10_PSP-1549', }, '9.1.0.13' =>{ 'GA-RUP' => '9.1.0.13_GA-RUP_2021-11_PSP-1650', 'DA-RUP' => '9.1.0.13_DA-RUP_2021-11_PSP-1651', }, '9.1.0.14' =>{ 'GA-RUP' => '9.1.0.14_GA-RUP_2021-12_PSP-1722', 'DA-RUP' => '9.1.0.14_DA-RUP_2021-12_PSP-1723', }, '9.2.0.0' => undef, '9.2.0.1' => { 'GA-RUP' => '9.2.0.1_GA-RUP_2021-06_PSP-1322', }, '9.2.1.0' => undef, '9.2.1.1' => { 'GA-RUP' => '9.2.1.1_GA-RUP_2021-06_PSP-1313', }, '9.2.1.2' => { 'GA-RUP' => '9.2.1.2_GA-RUP_2021-07_PSP-1246', 'DA-RUP' => '9.2.1.2_DA-RUP_2021-07_PSP-1247', }, '9.2.1.3' => { 'GA-RUP' => '9.2.1.3_GA-RUP_2021-08_PSP-1394', 'DA-RUP' => '9.2.1.3_DA-RUP_2021-08_PSP-1395', }, '9.2.1.4' => { 'GA-RUP' => '9.2.1.4_GA-RUP_2021-09_PSP-1432', 'DA-RUP' => '9.2.1.4_DA-RUP_2021-09_PSP-1433', }, '9.2.1.5' => { 'GA-RUP' => '9.2.1.5_GA-RUP_2021-10_PSP-1525', 'DA-RUP' => '9.2.1.5_DA-RUP_2021-10_PSP-1526', }, '9.2.1.6' => { 'GA-RUP' => '9.2.1.6_GA-RUP_2021-11_PSP-1640', 'DA-RUP' => '9.2.1.6_DA-RUP_2021-11_PSP-1641', }, '9.2.1.7' => { 'GA-RUP' => '9.2.1.7_GA-RUP_2021-12_PSP-1698', 'DA-RUP' => '9.2.1.7_DA-RUP_2021-12_PSP-1699', }, '9.3.0.0' => undef, '9.3.0.1' => { 'GA-RUP' => '9.3.0.1_GA-RUP_2021-11_PSP-1764', }, '9.3.0.2' => { 'GA-RUP' => '9.3.0.2_GA-RUP_2021-12_PSP-1643', 'DA-RUP' => '9.3.0.2_DA-RUP_2021-12_PSP-1644', }, '9.3.0.3' => { 'GA-RUP' => '9.3.0.3_GA-RUP_2022-01_PSP-1724', 'DA-RUP' => '9.3.0.3_DA-RUP_2022-01_PSP-1725', }, ); ### # CodeLinks: File links and sha256 checksum values for those files our %CodeLinks = ( # Code Bundles 'Isilon OneFS 9.3.0.3 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.3.0.3_Install_Bundle_2022-01.tgz', 'sha256' => '1a9434347f315af60a71428cf161dcc95771d44beef26ef15ec19de751d7ef03', }, 'Isilon OneFS 9.3.0.2 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.3.0.2_Install_Bundle_2021-12.tgz', 'sha256' => '32d67df5dfc5133f7899417058733d20f333d3bf008d4568d971e62a37410153', }, 'Isilon OneFS 9.3.0.1 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.3.0.1_Install_Bundle_2021-11.tgz', 'sha256' => '419deffd5d7e1945de45ea7af168db2d3b65d1c482ae6b7d84facfbe101633c7', }, 'Isilon OneFS 9.2.1.7 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.7_Install_Bundle_2021-12.tgz', 'sha256' => 'b9625509273264bba8dbaa36a0ff221e6336ca65df16d88e99b0c642ab5b5c24', }, 'Isilon OneFS 9.2.1.6 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.6_Install_Bundle_2021-11.tgz', 'sha256' => '1b7fae95e9b61000d6fb9f891c29fb0ca7bf47d3f03efa68194e22d2fd9bb7ac', }, 'Isilon OneFS 9.2.1.5 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.5_Install_Bundle_2021-10.tgz', 'sha256' => '818f9278d96ff4004b09d877835289f9a230bbd4c2bb5dcfa919261d919b9b83', }, 'Isilon OneFS 9.2.1.4 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.4_Install_Bundle_2021-09.tgz', 'sha256' => 'd76b675c457c141b3d163174dd012b70afb2ba64db017645dc0050f8417fbf3d', }, 'Isilon OneFS 9.2.1.3 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.3_Install_Bundle_2021-08.tgz', 'sha256' => 'a2a8b32e1d2695f11556475fad445edcbca8f7a0c93dec1446fc69b66970a601', }, 'Isilon OneFS 9.2.1.2 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.2_Install_Bundle_2021-07.tgz', 'sha256' => '0513a4370d10589b85121dc2276fc3a8e8fe417079e2009fc154a6d2c1544ce9', }, 'Isilon OneFS 9.2.1.1 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.1.1_Install_Bundle_2021-06.tgz', 'sha256' => 'b43f219c78d0468234a1116cfd68b5a16f56e47e82acb0e6b71f115dbb852db2', }, 'Isilon OneFS 9.2.0.1 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.2.0.1_Install_Bundle_2021-06.tgz', 'sha256' => '8dd2f09616bca97da708d83462c43349b29465079297812852c85decbd491f3f', }, 'Isilon OneFS 9.1.0.14 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.14_Install_Bundle_2021-12.tgz', 'sha256' => '54cc0b805eceac8dfc8bd1cbd1f72a27cce311115ddcc6ac403085c12cf7ec93', }, 'Isilon OneFS 9.1.0.13 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.13_Install_Bundle_2021-11.tgz', 'sha256' => 'f48bad14d19addb0d0ededd7ce650fefb7cb584b35b35d4f34ea1d1486bcc59c', }, 'Isilon OneFS 9.1.0.12 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.12_Install_Bundle_2021-10.tgz', 'sha256' => 'dcd925ca95c40dc7d9e8e0f2c94a89e45a68f9259bf1c21015145d86a2eec485', }, 'Isilon OneFS 9.1.0.11 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.11_Install_Bundle_2021-09.tgz', 'sha256' => '551910017388a60ea3694664dc84000b64bc252f848d4003e1ef41b575172d29', }, 'Isilon OneFS 9.1.0.10 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.10_Install_Bundle_2021-08.tgz', 'sha256' => '6d33ee648961f5b88f81f9c54f2c45b7767f1afab533ca83e483a107d9afc9fc', }, 'Isilon OneFS 9.1.0.9 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.9_Install_Bundle_2021-07.tgz', 'sha256' => '76d516abb41f768966eaf6c1224c268921d9f61a97c7d71153cefb24c19253c5', }, 'Isilon OneFS 9.1.0.8 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.8_Install_Bundle_2021-06.tgz', 'sha256' => '967947100d8d49a5d687f4cc3e8f27ac1050b83919561549c9afbfbf18ea29c1', }, 'Isilon OneFS 9.1.0.7 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.7_Install_Bundle_2021-05.tgz', 'sha256' => 'a8eab39b98ccd5c199ee7e690de2f0b2b3c4926b7566aca7a564f8ed0b604fe9', }, 'Isilon OneFS 9.1.0.6 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.6_Install_Bundle_2021-04.tgz', 'sha256' => 'a04b04beea9a5e1864e02536bfefbe16e33271b4cc33bb80a704e4c3a84d999c', }, 'Isilon OneFS 9.1.0.5 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.5_Install_Bundle_2021-03.tgz', 'sha256' => '69c181ea5f56252d0dd118e9c33fabe172fdda3f7daba2e19f3ba622c9bb9b0b', }, 'Isilon OneFS 9.1.0.4 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.4_Install_Bundle_2021-02.tgz', 'sha256' => '25ff9763b4b40996d5a4434dd55b1902cc5fe03b280701585655e9eacd85b722', }, 'Isilon OneFS 9.1.0.3 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.3_Install_Bundle_2021-01.tgz', 'sha256' => '83a0528684a081bfdc7d4d3c1abb2df813bc6f1591a8badd7bb07fdb3524ded2', }, 'Isilon OneFS 9.1.0.2 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.2_Install_Bundle_2020-12.tgz', 'sha256' => '69d901c21aa8d2243efcc7266713166f145aa57bba5d3338f78fadc99fb8259e', }, 'Isilon OneFS 9.1.0.1 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.1_Install_Bundle_2020-11.tgz', 'sha256' => '0bc1b3242ad3197d565626e1d14b480e96576d488e2c4ae6c87942bdad33d555', }, 'Isilon OneFS 9.0.0.0 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.0.0.0_Install_Bundle_2020-11.tgz', 'sha256' => 'ea57a760045ccd93f0180d04d1f9c4acf8643c736f7529bbe577a618fcca70d1', }, 'Isilon OneFS 8.2.2.0 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.2.2.0_Install_Bundle_2021-11.tgz', 'sha256' => '0cb5504d459972de30ab781c1e668c5c9c782febdfacdb9a1fde23ebeeef2754', }, 'Isilon OneFS 8.2.1.0 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.2.1.0_Install_Bundle_2021-01.tgz', 'sha256' => '842f7052e3ac6940033ae4b308a838bd1d92282377e7dd2e289d571afe8ad861', }, 'Isilon OneFS 8.2.0.0 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.2.0.0_Install_Bundle_2020-06.tgz', 'sha256' => '4c751260cb36283f7e7f643e5a2beb9e8fd20ad7e39c9c2c7455fb90c985ff7b', }, 'Isilon OneFS 8.1.2.0 installation bundle' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.1.2.0_Install_Bundle_2021-06.tgz', 'sha256' => '2d0e3160a67186c5c15a71a104345bf53c7bc979618a32eadc0eeac71f353d9e', }, # Support Tools Package '8.X_GA-TOOLS_2021-05_PSP-1034' => { 'Code' => 'https://dl.dell.com/downloads/DL102777', 'sha256' => '017536fe54a360c8fff6ba58bea8f3183b3ccaf8982c131f90f00492f4725101', }, '9.X_GA-TOOLS_2021-10_PSP-1695' => { 'Code' => 'https://dl.dell.com/downloads/DL104382', 'sha256' => 'fb017892af561780f01e77ea30b65c775d2f24f1b1ac94896d52fc1af58cb585', }, # Health Check Packages 'HealthCheck_9.2.1_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104187', 'sha256' => 'df7a8750fb4ae7ca51b17aa038997a6cb24546ebb181087a733f8ac9c4617e8f', }, 'HealthCheck_9.2.0_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104171', 'sha256' => 'b7b5bb0d4a933feda0b1b21fcec40d344a678933c8fa26954c57ec967d426518', }, 'HealthCheck_9.1.0_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104172', 'sha256' => 'd427899b708aa5e12788b376227b025399e96b1ff82b6e6120de12ad27182a1d', }, 'HealthCheck_9.0.0_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104173', 'sha256' => '1fb585ff84073f4fa54df45a3b72ebdda33a227d15c8f61e3bf4c72e4e18c589', }, 'HealthCheck_8.2.2_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104174', 'sha256' => 'e5d577dd0fbc51a6900e901cd998203164f313078576a493dbe176534f9813cc', }, 'HealthCheck_8.2.1_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104175', 'sha256' => '089ed6322b9d4ac6c0cb27a7f9cf171b6015fb5d2945b8fbbba1ce7a16011ec5', }, 'HealthCheck_8.2.0_2021-09' => { 'Code' => 'https://dl.dell.com/downloads/DL104156', 'sha256' => '7a212183816eeb062a86302e33d1bfd3086c8d9d4e12fbe565affc70e8518bd9', }, 'HealthCheck_8.1.3_2021-05' => { 'Code' => 'https://dl.dell.com/downloads/DL104154', 'sha256' => 'b1fb5dbe98660cb455766e07667982c79444a1af1fb196986588825d4e1eeb36', }, 'HealthCheck_8.1.2_2021-05' => { 'Code' => $DL_EOSL, 'sha256' => 'e2b8adfd0683a085b5999a77916ba9a879b520674ecf42bf2bf232b31e330298', }, 'HealthCheck_8.1.0_2021-05' => { 'Code' => 'https://dl.dell.com/downloads/DL104152', 'sha256' => 'ce296bc66b241bd8d77d1df0847d0575d329ab800c1df449e323012b8c8ec814', }, 'HealthCheck_8.0.0.7_2020-02' => { 'Code' => 'https://dl.dell.com/downloads/DL97357', 'sha256' => '7bb421f9996fb995c3f802891f54d33bb266713c405449cc2eb8330691dd8537', }, # Node Firmware Packages 'Isilon Node Firmware Package 11.2' => { 'Code' => 'https://dl.dell.com/downloads/DL99081', 'sha256' => '2d345cc9ce5d896390f7bbe41779975bd4cca4e25eaa0f77f739f0b90d36ff56', }, 'Isilon Node Firmware Package 10.3.6' => { 'Code' => 'https://dl.dell.com/downloads/DL94898', 'sha256' => '510c14a7c540c63c83949153b470cb871705d68b396cb9dae1de083259eca7ea', }, 'Isilon Node Firmware Package 9.3.5' => { 'Code' => 'https://dl.dell.com/downloads/DL83480', 'sha256' => 'f03d11fd5d8e909b7a71e6e2393234cc3a35599cca056dc35aca0fe72232dcf9', }, # BMC Firmware Packages 'Isilon BMC Firmware Package 1.0' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/firmware/IsiFw_Package_BMC_v1.0.tar', 'sha256' => '144222e82d594b9239873d32bbbfaa1f2557e4031196f0b0d8141f2860e4bc3b', }, # Drive Support Packages 'Isilon Drive Support Package 1.39.1' => { 'Code' => 'https://dl.dell.com/downloads/DL92292', 'sha256' => '61acee58cd105c8d5cc8d80cc8a5dfb8a226012a5e5cc9c1bd99d64040fc6a77', }, # FRU Package 'IsiFru Package' => { 'Code' => 'ftp://ftp.emc.com/outgoing/Fru_Package/IsiFru_Package_201904111419.tgz', 'sha256' => 'a13bba4691901fac4015d4d2bad5b40e0d9ccdc29d3b3ff0a374d606c3f57635', }, # Scripts 'NFS Export Security Repair Script' => { 'Code' => 'ftp://ftp.emc.com/outgoing/support/isi_nfs_export_security_repair_script/isi_nfs_export_security_repair', 'sha256' => '7d5a18a47f10b6a326be83e141a28cad1f7c14ed2098d741d4b12d41963b9f54', }, # Patches # 9.3.0 RUPs '9.3.0.3_GA-RUP_2022-01_PSP-1724' => { 'Code' => 'https://dl.dell.com/downloads/DL107584', 'sha256' => '88ce68d0711d600958503159b9e8e5d893cb1b6f2c711f86cfa3213ce1c88bfb', }, '9.3.0.3_DA-RUP_2022-01_PSP-1725' => { 'Code' => $DL_DA, 'sha256' => '9b07e1d30bea8e3042c59cecd081d7b6c981571058e734e43cfcc6d0be5ded24', }, # 9.2.1 RUPs '9.2.1.7_GA-RUP_2021-12_PSP-1698' => { 'Code' => 'https://dl.dell.com/downloads/DL107353', 'sha256' => 'e65871f93fed37bd43103460ab18a36d99cb7eff6c1799b76ae3b8d7a607a6c5', }, '9.2.1.7_DA-RUP_2021-12_PSP-1699' => { 'Code' => $DL_DA, 'sha256' => '592da70a655be393ba78234758a7a5183423216be6d792315c038fe96a029b8f', }, # 9.2.0 RUPs '9.2.0.1_GA-RUP_2021-06_PSP-1322' => { 'Code' => 'https://dl.dell.com/downloads/DL104541', 'sha256' => '169ad5f046c2a231a12b24c7540b82369d7438ef2bbe65c3493151a8d7b0a1f6', }, # 9.1.0 RUPs '9.1.0_UGA-PATCH-INFRA_2021-12_PSP-902' => { 'Code' => 'https://dl.dell.com/downloads/DL107492', 'sha256' => '33d34fb3abc93a89ee5a4427c410ae3eabcb8a4aa4f6fb8c1ff3cdfd3508cd1c', }, '9.1.0.14_GA-RUP_2021-12_PSP-1722' => { 'Code' => 'https://dl.dell.com/downloads/DL107427', 'sha256' => '872770add86546437dfcfad7a793329dcc4d30fac375c551df85e89b144090e3', }, '9.1.0.14_DA-RUP_2021-12_PSP-1723' => { 'Code' => $DL_DA, 'sha256' => '8a90d59b8186ecc6ca5975cef948267b5d3ac73a2cdec55672c28633d5ea1c66', }, # 9.0.0 RUPs '9.0.0_GA-RUP_2020-11_PSP-406' => { 'Code' => 'https://dl.dell.com/downloads/DL101293', 'sha256' => '8f7bd5b32934404dd7d90806bf853be4a95d8e769e10e7416c4565a89d3739e1', }, '9.0.0_DA-RUP_2020-11_PSP-407' => { 'Code' => $DL_DA, 'sha256' => 'd72f7c65194fa87467aeff9811030b759a14523cab66af6bc5e5680bb4962f2f', }, # 8.2.2 RUPs '8.2.2_DA-RUP_2021-09_PSP-1687' => { 'Code' => $DL_DA, 'sha256' => 'dc0287e61948391ebdd16ce8d4ef8879a8298858a652b37f9bbabe666d970493', }, '8.2.2_GA-RUP_2021-11_PSP-1571' => { 'Code' => 'https://dl.dell.com/downloads/DL107022', 'sha256' => 'e326db2e6e73a00bfab791876da513f20b5b0aa8d7888513a32e471204201292', }, # 8.2.1 RUPs '8.2.1_GA-RUP_2021-01_PSP-397' => { 'Code' => 'https://dl.dell.com/downloads/DL102404', 'sha256' => '5dd0fe2a20257d5368ce364a6396e1da9c51c7575f88a0f2af4baf5691d49165', }, '8.2.1_DA-RUP_2021-01_PSP-590' => { 'Code' => $DL_DA, 'sha256' => '11191694e75c96cc29185776bdc76e688d697a901eef5d3ed3278dcb2a80b253', }, # 8.2.0 RUPs '8.2.0_UGA-PATCH-INFRA_2019-08_251917' => { 'Code' => 'https://dl.dell.com/downloads/DL95238', 'sha256' => 'fc595aea778f99fda4a16221fc89f02027ff01304610fdc496394adc88b59502', }, '8.2.0_KGA-RUP_2020-03_270126' => { 'Code' => 'https://dl.dell.com/downloads/DL97887', 'sha256' => '7e00172b05737b0d4be1ba5b7c33e7c0d97d279274170918104f6c05897ae83f', }, '8.2.0_UDA-RUP_2020-02_264782' => { 'Code' => $DL_DA, 'sha256' => '41e8b257d2053487c2323d539aae9f50ceeb1da67de85028480a1c9ba4ecc199', }, '8.2.0_GA-RUP_2020-06_273332' => { 'Code' => 'https://dl.dell.com/downloads/DL99183', 'sha256' => 'd878d47bc4770aae3ea65752791fccdcfb315406a561908ca0356af8dcf88cec', }, # 8.1.2.0 RUPs '8.1.2.0_DA-RUP_2021-06_PSP-1195' => { 'Code' => $DL_DA, 'sha256' => 'eb4bfb8c9093628b797a48d93f1527dcadbb96ad0fdd858222b6239e48a95bb1', }, '8.1.2.0_GA-RUP_2021-06_PSP-1194' => { 'Code' => $DL_EOSL, 'sha256' => '366a51acf3614cb2e20561e9eaf1162c4d6743f28bd521c9c9f9c8949e764eee', }, '8.1.2.0_UGA-PATCH-INFRA_2019-09_255624' => { 'Code' => 'https://dl.dell.com/downloads/DL95608', 'sha256' => 'd7965ed922da0bd3790e09b4a0b05ad52ffde36f9470a47d498c622d81d20633', }, '8.1.2.0_UGA-PATCH-INFRA_2020-08_PSP-268' => { 'Code' => 'https://dl.dell.com/downloads/DL99885', 'sha256' => 'f3b0fe52eec11471faf9d7be903c44400a6a67feda595fb0d15eda32ae736991', }, # 8.1.0 RUPs '8.1.0.4_KGA-RUP_2019-09_256245' => { 'Code' => 'https://dl.dell.com/downloads/DL95835', 'sha256' => '188243439bd352b4060130476e52f4bcb88aa99c066f3f349fe6c9bedb55f5a6', }, '8.1.0.4_UDA-RUP_2019-09_256244' => { 'Code' => $DL_DA, 'sha256' => '7dc46c54d92c4b27004b6b1f5f0d3dbdb0788da2cc24211e750184b4cb4a4814', }, '8.1.0.4_UGA-RUP_2019-09_256243' => { 'Code' => 'https://dl.dell.com/downloads/DL95834', 'sha256' => 'dfa14d8ceadb95455e1668746c620161745900ff13f189af166d46307680503b', }, ### 'patch-240796' => { 'Code' => 'https://dl.dell.com/downloads/DL92411', 'sha256' => '03d6b3211fc14f4e705c1cac551ec953473fe3ab69b6543dbad0f5ed871eb55d', }, 'patch-240795' => { 'Code' => 'https://dl.dell.com/downloads/DL92147', 'sha256' => '46af020fedc862c70e83cd2b60bbbdcdcb6ab0325bb93d9aae97044fa220020f', }, 'patch-231386' => { 'Code' => 'https://dl.dell.com/downloads/DL90411', 'sha256' => '84276a3f1c2a97aa9bc784944bda2229032c4df00e1471691898eb4e7240b3c8', }, 'patch-229160' => { 'Code' => 'https://dl.dell.com/downloads/DL90208', 'sha256' => '9b20e01f7a87b1950c51399c11c2c9f805c50cc95d6752f401836ae10f58cafe', }, 'patch-229158' => { 'Code' => 'https://dl.dell.com/downloads/DL90209', 'sha256' => 'fd8bce193555161b1bf0522768463e74229e37f7b188be90aa10007a57ae5185', }, 'patch-229156' => { 'Code' => 'https://dl.dell.com/downloads/DL90210', 'sha256' => 'de4694c080ffb3048f1208175cc492cf426ffc0891f8ff59d0eb7948cfbfe79b', }, 'patch-228458' => { 'Code' => 'https://dl.dell.com/downloads/DL90409', 'sha256' => '2ba8ab8a5ab4e28e4e69b3b16c1719146e6a50ac81971c4466f0e522c5d6f826', }, 'patch-226984' => { 'Code' => 'https://dl.dell.com/downloads/DL89610', 'sha256' => '932ad12392556271217c7099fd0dca4eb3b87094f80a0bed0797e0361ab26b74', }, 'patch-226323' => { 'Code' => 'https://dl.dell.com/downloads/DL89668', 'sha256' => '724032a9f2bcafdd9a2027b5ffef6561340a1e1720524b9e31822c95c7ff5748', }, 'patch-224116' => { 'Code' => 'https://dl.dell.com/downloads/DL90410', 'sha256' => '9420b627b7dd419b93a7fed275948e1414640eaee9a35b2ad73238d38c5bf72f', }, 'patch-223338' => { 'Code' => 'https://dl.dell.com/downloads/DL90474', 'sha256' => 'faf53ca44e03292266f00e211409acb2076c000bdc5ee89d3010ad1ef35dfb0a', }, 'patch-223311' => { 'Code' => 'https://dl.dell.com/downloads/DL90081', 'sha256' => 'fa11cb91d73266a22fc9950cba85715ec7e2d72d16e4734d353cc51f337d92c6', }, 'patch-222487' => { 'Code' => 'https://dl.dell.com/downloads/DL89176', 'sha256' => 'b1e9ecafb4f793cd00e84947b5e6672bc60d984eb5501f26d01f7872ea3ee106', }, 'patch-221949' => { 'Code' => 'https://dl.dell.com/downloads/DL89175', 'sha256' => '5e280e5bfaa877e8217418128d82553ad60787e037dd1caef79403fcc1e571a1', }, 'patch-221947' => { 'Code' => 'https://dl.dell.com/downloads/DL89667', 'sha256' => 'bf3a53f4b490bcc67faf07fbd0fe84d01f199e74f892cde46978eeca9de5c27b', }, 'patch-221946' => { 'Code' => 'https://dl.dell.com/downloads/DL88631', 'sha256' => 'ac14bd365452176a4c2b25f5dbd1fbaa1415a4da5f352d5566b89a597ad8b8c2', }, 'patch-221638' => { 'Code' => 'https://dl.dell.com/downloads/DL89137', 'sha256' => '21f0a7d02d619c9f0f9ea9dda9b03d2bf3afa1d812c909471857ef49644a222a', }, 'patch-221617' => { 'Code' => 'https://dl.dell.com/downloads/DL88630', 'sha256' => 'e0e0b4a80a499dcb8d7c2f66b2aecdeb89a0b9aef9ba0603eeaeea7ee188dced', }, 'patch-221603' => { 'Code' => 'https://dl.dell.com/downloads/DL89174', 'sha256' => 'd5164a061d275713a4ce899ab83ef595aba49470289e7ae3d07f0f5dae68b131', }, 'patch-221602' => { 'Code' => 'https://dl.dell.com/downloads/DL88628', 'sha256' => '3e9b8b8c79bd24912f22ae52fa27c4b076f924d486a22f180c969fdc3116717a', }, 'patch-220176' => { 'Code' => 'https://dl.dell.com/downloads/DL89173', 'sha256' => '5d7a367db9ce02c9e0816db232dc127646a05eb232a61c7208c806617d409662', }, 'patch-220098' => { 'Code' => 'https://dl.dell.com/downloads/DL89485', 'sha256' => 'cb5cdcd1cad22828e722eb0f1a163e9f67fa57271c908ed1d7b5820bdfc9363f', }, 'patch-219279' => { 'Code' => 'https://dl.dell.com/downloads/DL89676', 'sha256' => 'f5e86c749d1de7fcc0a9b8ee56cb1dc79bf2e71b90cb07fa3a916d6a6e728b9f', }, 'patch-217638' => { 'Code' => 'https://dl.dell.com/downloads/DL88034', 'sha256' => '0be46fabcf5e4c43f0623bdb64a2502a58eab8615590817e5708c8cf0124cb74', }, 'patch-217637' => { 'Code' => 'https://dl.dell.com/downloads/DL87915', 'sha256' => '219c8b0389d63313d59e3eebba96bcc45f34a8b2f1f638ba1ca239052b39d99c', }, 'patch-213287' => { 'Code' => 'https://dl.dell.com/downloads/DL88627', 'sha256' => '9c460a6ec57fbea5f7b369f8a31b97f5494b370bc97ee000997c36804d9eb5fc', }, 'patch-213286' => { 'Code' => 'https://dl.dell.com/downloads/DL88626', 'sha256' => '5c537b089a2da896328e5a8c8d4930952adeb2d68063468ee71fff7ecd75fbd1', }, 'patch-213281' => { 'Code' => 'https://dl.dell.com/downloads/DL88171', 'sha256' => '61591a2f2c51c8c726e1d2f0c4d40bbfec12ceb78cc02ec8851d3bdfbe79f806', }, 'patch-213280' => { 'Code' => 'https://dl.dell.com/downloads/DL88003', 'sha256' => 'a9bc5f40f689c33b59a50641c7fcbab0fa9b54a39f74ac83217189092c1b2603', }, 'patch-213279' => { 'Code' => 'https://dl.dell.com/downloads/DL88249', 'sha256' => '13191be36b27be6e08b9b5fc44624b351241edd490aa0bb111290047226f9260', }, 'patch-211361' => { 'Code' => 'https://dl.dell.com/downloads/DL87018', 'sha256' => '14c96e73af2dc9e39c62f93ce7ab004cda3b66a9d2d133877473d0aeffc1b187', }, 'patch-206322' => { 'Code' => 'https://dl.dell.com/downloads/DL86823', 'sha256' => '6d99bfa5ee6f28b14871b6e2f5f5ac6b5a9122c3aefa15f73a609d526426fc32', }, 'patch-189133' => { 'Code' => 'https://dl.dell.com/downloads/DL83560', 'sha256' => '0bd5e5e8e1576483549bfd80195c1e578b07b195ff09c6106cfae87acfc59764', }, 'patch-189132' => { 'Code' => 'https://dl.dell.com/downloads/DL83559', 'sha256' => '12ad326888591a737ec394bc620f5401c563dc1bc3d76ba946e0ef898acc3731', }, 'patch-188660' => { 'Code' => 'https://dl.dell.com/downloads/DL82519', 'sha256' => '38285bce926eb33bf6ce2c0a314ecaec8712b58c44c29c133072a18d72bbe3e4', }, 'patch-188239' => { 'Code' => 'https://dl.dell.com/downloads/DL82813', 'sha256' => '2709ef574256096358625d0baec77723029d43bde4d65ea51c94b8e37c9ecbdf', }, 'patch-181490' => { 'Code' => 'https://dl.dell.com/downloads/DL80015', 'sha256' => 'a8e44b68f40a7d4c3e21ec3682e81e0d8b89001aacd6222cd93581c123fd2e05', }, 'patch-175451' => { 'Code' => 'https://dl.dell.com/downloads/DL71573', 'sha256' => '8318d05b00f00e01a0b916da9ca912b2b03ffba2cad6f5a939eb18d2c49fbf61', }, 'patch-164666' => { 'Code' => 'https://dl.dell.com/downloads/DL70637', 'sha256' => '25b53e98b9caf89c3b5866f8caf46b2e6d0b29148c3c53132fdeb4fa0afec8cf', }, 'patch-161166' => { 'Code' => 'https://dl.dell.com/downloads/DL61739', 'sha256' => '39614c55cbe5f4f9eb536b23bc67b411f07c36b65142f5951d55be8281ab31bf', }, 'patch-147686' => { 'Code' => 'https://dl.dell.com/downloads/DL59058', 'sha256' => 'e934289d49d2f722be22e4d750b5f78e4584f95b7cea582ec03f5245374317c9', }, 'patch-146974' => { 'Code' => 'https://dl.dell.com/downloads/DL58909', 'sha256' => '28fea8daea8e3588cce3b263a72cdfa49ebdfa553826483eb1688d65590ecd61', }, 'patch-141264' => { 'Code' => 'https://dl.dell.com/downloads/DL58028', 'sha256' => '3bd4a7268c0d2b14475567ed7e496beab7664ec98bc7a833b5dd9f6a7269a044', }, 'patch-141263' => { 'Code' => 'https://dl.dell.com/downloads/DL58027', 'sha256' => '7bb9fc6ca8d8a3133623da45cfdd21942082e37d9be156cbf3205f38717623a1', }, 'patch-141262' => { 'Code' => 'https://dl.dell.com/downloads/DL58026', 'sha256' => 'b2d9d3ae0b7b6aef3f83c08320023ae2f91a99caa0d30fd3d6df257fb9721eda', }, 'patch-134049' => { 'Code' => 'https://dl.dell.com/downloads/DL55811', 'sha256' => '544ea0986deb55e0142c2d4b33fed99292b59f6b0f357cea0c5905776527fdf3', }, 'patch-130491' => { 'Code' => 'https://dl.dell.com/downloads/DL55310', 'sha256' => '66ff73447d6041261f3d529b923b25b9bdd50b3533e63c0fb482a8c4359f5074', }, # OneFS installation files 'Isilon OneFS 9.1.0.0 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.1.0.0_Install.tar.gz', 'sha256' => '2680b559bbc6b1de671145fb4cc4c6b1f652f45207f843b0a0686ebb68cb870c', }, 'Isilon OneFS 9.0.0.0 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v9.0.0.0_Install.tar.gz', 'sha256' => '13a5178248868624849f57487e9d4ecc45c3b67cab79ada21b468cfe4e74ceb6', }, 'Isilon OneFS 8.2.2.0 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.2.2.0_Install.tar.gz', 'sha256' => '8cb0ccc94439a9c2758a42bcc3fdc22b09a985ccfd3455bb472fe4fae569577f', }, 'Isilon OneFS 8.2.1.0 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.2.1.0_4_Install.tar.gz', 'sha256' => '82e6c0ff74e2514600c267010d8eb70251692b60daa3354f866e57f1448eaf36', }, 'Isilon OneFS 8.2.0.0 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.2.0.0_Install.tar.gz', 'sha256' => 'aef29a3a37d0e17468f0e37382f123f8d4adf74913f0d717e4591ac29e79b88a', }, 'Isilon OneFS 8.1.3.0 installation file' => { 'Code' => $DL_DA, 'sha256' => '5c02c4165956565b5a32009b9752bf7270920c5538a7ff210991338096b5e306', }, 'Isilon OneFS 8.1.2.0 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.1.2.0_Install.tar.gz', 'sha256' => 'f2de544f6616cc681a7889090e5970a1ea3363f9e9e44a4fd1ca1b45ea9b3a3e', }, 'Isilon OneFS 8.1.0.4 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.1.0.4_Install.tar.gz', 'sha256' => '923444b5901340f3a9efe842ae974ba830492c4d26e0f12607f67b8d8f2762ab', }, 'Isilon OneFS 8.0.0.7 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v8.0.0.7_Install.tar.gz', 'sha256' => '64f7ccdc2506a631b0d0e2662df06420dd2c56ed2edf0faee1fb2eb8ac1fce0f', }, 'Isilon OneFS 7.2.1.6 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v7.2.1.6_Install.tar.gz', 'sha256' => '47d6f772ad1bf69ae9aaeb1443ae21f235d30b4e53a594412575167202b76185', }, 'Isilon OneFS 7.2.0.5 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v7.2.0.5_Install.tar.gz', 'sha256' => '2c195e27425e87543c188754f086a01a28188bbba40bf8860b9bc68db558a54d', }, 'Isilon OneFS 7.1.1.11 installation file' => { 'Code' => 'ftp://ftp.emc.com/pub/rcm/Isilon/onefs/OneFS_v7.1.1.11_Install.tar.gz', 'sha256' => '37d7dd688a9f6c038418a5b2959ff5f8fc490d5429eab605a3cc17d3e541502c', }, ); } # Activity durations # Base -> Time required to validate checksum and complete and pre/post # steps such as installing/uninstalling a firmware package # Drives/Nodes -> Time required per drive or node for updates to be performed # Regex Expression -> Used to add additional time based on regex criteria # These must be accomodated for in the getActivityDuration routine BEGIN { our %Durations = ( 'Drive Support Package' => { 'Base' => 15, 'Nodes' => 10, 'Drives' => 1.5, }, 'Node Firmware Update' => { 'Base' => 15, 'Nodes' => 10, '^(CMC|CMD|CAR)' => 7, '^(DE|EP)' => 3, 'Integrated Remote Access Controller' => 10, 'BIOS' => 10, 'Mellanox-CX' => 2, }, 'OneFS Upgrade' => { 'Base' => 25, 'Nodes' => 15, }, 'Reboot' => { 'Nodes' => 8, '^(S210|X410|X210|NL410)' => 3, '^(HD400)' => 10, '^(?:A2000?|A3000?|A250|A2050)' => 3, '^(?:H400|H500|H600|H7000?|H5600|H550|H5650)' => 3, '^(?:F800|F810|F850)' => 3, }, 'Patch' => { 'Base' => 10, 'Nodes' => 9, 'Service Restart' => 4.5, }, ); } BEGIN { our %ReleaseNotes = ( 'OneFS' => { '9.3.0' => 'https://dl.dell.com/content/docu105998', '9.2.1' => 'https://dl.dell.com/content/docu104030', '9.2.0' => 'https://dl.dell.com/content/docu103229', '9.1.0' => 'https://dl.dell.com/content/docu100463', '9.0.0' => 'https://dl.dell.com/content/docu98858', '8.2.2' => 'https://dl.dell.com/content/docu96915', '8.2.1' => 'https://dl.dell.com/content/docu95361', '8.2.0' => 'https://dl.dell.com/content/docu93613', '8.1.2' => 'https://dl.dell.com/content/docu90168', '8.1.1' => 'https://dl.dell.com/content/docu87518', '8.1.0' => 'https://dl.dell.com/content/docu84278', '8.0.1' => 'https://dl.dell.com/content/docu78727', '8.0.0' => 'https://dl.dell.com/content/docu65098', '7.2.1' => 'https://dl.dell.com/content/docu60143', '7.2.0' => 'https://dl.dell.com/content/docu56010', '7.1.1' => 'https://dl.dell.com/content/docu54294', }, 'DF' => 'https://dl.dell.com/content/docu64847', 'NF' => { '11.2' => 'https://dl.dell.com/content/docu98769', '10.3.6' => 'https://dl.dell.com/content/docu69984', '9.3.5' => 'https://dl.dell.com/content/docu84897', }, 'RUP' => { '9.2.1' => 'https://www.dell.com/support/kbdoc/000189200', '9.2.0' => 'https://www.dell.com/support/kbdoc/000189197', '9.1.0' => 'https://www.dell.com/support/kbdoc/000181149', '9.0.0' => 'https://www.dell.com/support/kbdoc/000157639', '8.2.2' => 'https://www.dell.com/support/kbdoc/000157589', '8.2.1' => 'https://www.dell.com/support/kbdoc/000157866', '8.1.2' => 'https://www.dell.com/support/kbdoc/000157717', } ); } $CHECKS{'checkDriveFirmware'}{'Description'} = 'Checks for out of date Drive Firmware and calls other related drive firmware checks'; sub checkDriveFirmware { if (wasCalled()) { return } checkDriveSupportPackage(); checkFCOF042415EE(); checkFCOF022318EE(); checkFCOF031617FC(); } # End checkDriveFirmware $CHECKS{'checkDriveSupportPackage'}{'Description'} = 'Checks for drive firmware updates available in the Drive Support Package'; sub checkDriveSupportPackage { return if wasCalled(); # Skip if no valid firmware version to check against return unsupported('No Drive Support Package selected') unless $RPS{'DSP'} and $RPS{'DSP'} ne 'None'; # Skip if the cluster is virtual return unsupported('Unsupported on virtual platforms') if isVirtual(); # Skip if there are no Bootflash nodes checkPartitions(); return unsupported('The Drive Support Package is not supported on older node types') unless $CLUSTER_INFO{'Bootflash'}; printCheck("Drive Support Package ($RPS{'DSP'})"); gatherCTO(); gatherDriveFirmware(); our %DriveSupport; # Generate a sorted list of drive support packages my @dsps = sort {compareVersions($b, $a)} keys %DriveSupport; my $requires_91 = 0; my $max_model_length = 30; # Analyze for out of date firmware foreach my $model (keys %{$CLUSTER_INFO{'Drive Firmware'}}) { my $model_length = length($model); if ($model_length > $max_model_length) { $max_model_length = $model_length } foreach my $firmware (keys %{$CLUSTER_INFO{'Drive Firmware'}{$model}}) { next unless $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; my $fw = $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; my $dsp_target = 0; foreach my $pkg (@dsps) { # Stop if we found a target last if $dsp_target; # Skip this package if no updates for this model next unless $DriveSupport{$pkg}{$model}; # There may be multiple target versions for each model my @targets = keys %{$DriveSupport{$pkg}{$model}}; # Check to see if the firmware is already at a target version my @results = grep {$_ eq $firmware} @targets; last if @results; # Skip this package if newer than requested next if compareVersions($pkg, $RPS{'DSP'}) > 0; foreach my $target (@targets) { # Check for firmware updates # Check if the current version is supported # Unqualified upgrade is allowed, skip current version check my @results = grep {$_ eq $firmware or $_ eq 'allow'} @{$DriveSupport{$pkg}{$model}{$target}}; if (@results) { # Firmware updates available $dsp_target = $target; # To perform unqualified upgrades requires OneFS 9.1+ if ($results[0] eq 'allow') { $requires_91 = 1 } last; } } } ### # Check for nodes that support DSP foreach my $lnn (keys %{$fw->{'Counts'}}) { # Only check on Bootflash nodes next unless $NODE_INFO{$lnn}{'Bootflash'}; # Add to the list of nodes push(@{$fw->{'DSP'}{'Nodes'}}, $lnn); # Add to the count of devices $fw->{'DSP'}{'Count'} += $fw->{'Counts'}{$lnn}; # Flag FRU Package if any CTO nodes require updating if ($dsp_target and $NODE_INFO{$lnn}{'CTO'}) { $RPS{'Drive Support Package'}{'FRU'} = 1; } } # Next firmware if no updates were found next unless $dsp_target and $fw->{'DSP'}{'Nodes'} and $fw->{'DSP'}{'Count'}; ### # Beyond this point drive firmware updates are available. debug("checkDriveSupportPackage: DSP $RPS{'DSP'} supports ". "updating from '$firmware' to '$dsp_target'"); # Set target firmware version $fw->{'DSP'}{'Target'} = $dsp_target; # Add to the list of nodes to be updated @{$RPS{'Drive Support Package'}{'Nodes'}} = merge(@{$RPS{'Drive Support Package'}{'Nodes'}}, @{$fw->{'DSP'}{'Nodes'}}); # Add to the count of drives the drives to be updated $RPS{'Drive Support Package'}{'Drives'} += $fw->{'DSP'}{'Count'}; } } # Print out drive firmware information my $SPF_SUMMARY = "%-${max_model_length}s %-10s %-10s %-5s %s"; details(sprintf($SPF_SUMMARY, 'Model', 'Current', 'Desired', 'Count', 'Nodes')); foreach my $model (sort keys %{$CLUSTER_INFO{'Drive Firmware'}}) { foreach my $firmware (keys %{$CLUSTER_INFO{'Drive Firmware'}{$model}}) { my $fw = $CLUSTER_INFO{'Drive Firmware'}{$model}{$firmware}; next unless ($fw->{'DSP'}{'Nodes'} and $fw->{'DSP'}{'Count'}); details(sprintf($SPF_SUMMARY, $model, $firmware, ($fw->{'DSP'}{'Target'} or '-'), $fw->{'DSP'}{'Count'}, compressRange(@{$fw->{'DSP'}{'Nodes'}}))) } } if (!$RPS{'Drive Support Package'}{'Nodes'}) { # Drive firmware updates not available # Add the latest Drive Support Package to be used post OneFS upgrade if (isOneFSUpgrade() and isDestinationOneFS('>=9.1')) { addRequiredFiles("Isilon Drive Support Package $RPS{'DSP'}"); } printResults(); return; } $RPS{'Drive Support Package'}{'Target'} = "Drive Support Package $RPS{'DSP'}"; if ($RPS{'Drive Support Package'}{'FRU'}) { # Flag if the FRU package is required info('IsiFru Package is required'); } info(scalar(@{$RPS{'Drive Support Package'}{'Nodes'}}). ' nodes have out of date drive firmware: '. compressRange(@{$RPS{'Drive Support Package'}{'Nodes'}})); # Flag when OneFS 9.1+ is required for firmware updates if ($requires_91 and isCurrentOneFS('<9.1')) { critical('Drive firmware updates will require OneFS 9.1 or later to perform'); } # Flag an issue that requires nodes to be rebooted following firmware updates if ($RPS{'Drive Firmware'}{'KB490849'} and (isCurrentOneFS('<=8.0.0.4') or isCurrentOneFS(qr/^(8\.0\.1\.[01])$/))) { critical(scalar(@{$RPS{'Drive Firmware'}{'KB490849'}}). ' nodes are affected by KB490849: '. compressRange(@{$RPS{'Drive Firmware'}{'KB490849'}}). '. In versions OneFS 8.0.1.0, OneFS 8.0.1.1, or OneFS 8.0.0.4 and older, whenever nodes with affected Seagate drive types are updated using DSP 1.13 and newer, they must be rebooted after being updated. This can be done standalone or by any activity that reboots the nodes.'); } # Flag reasons why drive firmware updates should be a priority if ($RPS{'ETA'}{'193819'}{'Nodes'}) { my @nodes = (); foreach my $lnn (@{$RPS{'ETA'}{'193819'}{'Nodes'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Support Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). ' nodes are affected by ETA 193819: '. compressRange(@nodes); warning($message); push(@{$RPS{'Drive Support Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } if ($RPS{'Drive Firmware'}{'KB88930'}) { my @nodes = (); foreach my $lnn (@{$RPS{'Drive Firmware'}{'KB88930'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Support Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). ' nodes are affected by KB88930: '. compressRange(@nodes); warning($message); push(@{$RPS{'Drive Support Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } if ($RPS{'Drive Firmware'}{'KB502084'}) { my @nodes = (); foreach my $lnn (@{$RPS{'Drive Firmware'}{'KB502084'}}) { my @results = grep {$_ == $lnn} @{$RPS{'Drive Support Package'}{'Nodes'}}; @nodes = merge(@nodes, @results); } if (@nodes) { my $message = scalar(@nodes). ' nodes are affected by KB502084: '. compressRange(@nodes); warning($message); push(@{$RPS{'Drive Support Package'}{'Priority'}}, "Drive firmware updates are strongly recommended because: $message"); } } # Remove Drive Firmware Updates if requested only priority updates and there are no priority updates needed. if ($RPS{'Priority DF Only'} and $RPS{'Drive Support Package'}{'Nodes'} and !$RPS{'Drive Support Package'}{'Priority'}) { delete $RPS{'Drive Support Package'}; printResults(); return; } # Define support for DSP on current or target version my $supports_dsp = (isCurrentOneFS('>=7.1.1'))?1:0; my $supports_dsp_ndfu = ($supports_dsp and !isCurrentOneFS(qr/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/))?1:0; my $supports_dsp_sim = ($supports_dsp and !$supports_dsp_ndfu)?1:0; my $target_supports_dsp = isDestinationOneFS('>=7.1.1')?1:0; my $target_supports_dsp_ndfu = ($target_supports_dsp and !isDestinationOneFS(qr/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/))?1:0; my $target_supports_dsp_sim = ($target_supports_dsp and !$target_supports_dsp_ndfu)?1:0; # Select which position to perform drive firmware updates if ($supports_dsp_ndfu) { $RPS{'Drive Support Package'}{'Position'} = 1; } elsif ($target_supports_dsp_ndfu) { $RPS{'Drive Support Package'}{'Position'} = 3; } elsif ($supports_dsp_sim) { $RPS{'Drive Support Package'}{'Position'} = 2; $RPS{'Drive Support Package'}{'Simultaneous'} = 1; addCustomerInformation('Upgrade Plan', info("Drive firmware updates using Drive Support Package $RPS{'DSP'} when performed on OneFS $CLUSTER_INFO{'OneFS'} must be performed in the DSP/Simultaneous manner, rebooting nodes one at a time but updating all drives in that node in a simultaneous manner")); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "Drive firmware updates should be completed using DSP/Simultaneous method when performed on OneFS $CLUSTER_INFO{'OneFS'}"); } elsif ($target_supports_dsp_sim) { $RPS{'Drive Support Package'}{'Position'} = 3; $RPS{'Drive Support Package'}{'Simultaneous'} = 1; addCustomerInformation('Upgrade Plan', info("Drive firmware updates using Drive Support Package $RPS{'DSP'} when performed on OneFS $RPS{'Target'} must be performed in the DSP/Simultaneous manner, rebooting nodes one at a time but updating all drives in that node in a simultaneous manner")); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, "Drive firmware updates should be completed using DSP/Simultaneous method when performed on OneFS $RPS{'Target'}"); } else { addCustomerInformation('Upgrade Plan', info("Drive Support Package is not supported for drive firmware updates on OneFS $CLUSTER_INFO{'OneFS'}. Consider performing a OneFS upgrade to a version which supports the latest Drive Support Package.")); delete $RPS{'Drive Support Package'}; } # Accomodate for reboots required after using DSP and impacted by KB490849 if ($RPS{'Drive Support Package'} and $RPS{'Drive Firmware'}{'KB490849'} and !$RPS{'Drive Support Package'}{'Simultaneous'} and (isCurrentOneFS('<=8.0.0.4') or isCurrentOneFS(qr/^(8\.0\.1\.[01])$/))) { # Must have drives identified as affected, be running an affected OneFS version, and not performing DSP Simultaneous if (!isDestinationOneFS()) { # Not upgrading so definately affected addCustomerInformation('Upgrade Plan', 'Due to issues outlined in KB490849, nodes must be rebooted after performing drive firmware updates using the Drive Support Package.'); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, 'The subsequent reboot activity must be performed in the same maintenance window as the drive firmware update (KB490849)'); if (!$RPS{'Proactive Reboot'}) { $RPS{'Drive Firmware Reboot'}{'Position'} = '1'; @{$RPS{'Drive Firmware Reboot'}{'Nodes'}} = @{$RPS{'Drive Firmware'}{'KB490849'}}; @{$RPS{'Drive Firmware Reboot'}{'Additional Comments'}} = ('Only the following nodes require rebooting: '.compressRange(@{$RPS{'Drive Firmware'}{'KB490849'}})); } } elsif ($RPS{'Drive Support Package'}{'Position'} == 3 and $target_supports_dsp_ndfu and (isDestinationOneFS('<=8.0.0.4') or isDestinationOneFS(qr/^8\.0\.1\.[01]$/))) { # Upgrading to a version that is affected addCustomerInformation('Upgrade Plan', 'Due to issues outlined in KB490849, nodes must be rebooted after performing drive firmware updates using the Drive Support Package.'); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, 'The subsequent reboot activity must be performed in the same maintenance window as the drive firmware update (KB490849)'); $RPS{'Drive Firmware Reboot'}{'Position'} = '3'; @{$RPS{'Drive Firmware Reboot'}{'Nodes'}} = @{$RPS{'Drive Firmware'}{'KB490849'}}; @{$RPS{'Drive Firmware Reboot'}{'Additional Comments'}} = ('Only the following nodes require rebooting: '.compressRange(@{$RPS{'Drive Firmware'}{'KB490849'}})); } elsif ($RPS{'Drive Support Package'}{'Position'} < 3) { # Upgrading and running drive firmware updates on affected version prior to the OneFS upgrade addCustomerInformation('Upgrade Plan', 'Due to issues outlined in KB490849, nodes must be rebooted after performing drive firmware updates using the Drive Support Package.'); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, 'The subsequent reboot or OneFS activity must be performed in the same maintenance window as the drive firmware update (KB490849)'); } else { } } # Append files required for the updates push(@{$RPS{'Drive Support Package'}{'Files'}}, "Isilon $RPS{'Drive Support Package'}{'Target'}") if $RPS{'Drive Support Package'}{'Target'}; push(@{$RPS{'Drive Support Package'}{'Files'}}, 'IsiFru Package') if $RPS{'Drive Support Package'}{'FRU'}; info('Refer to KB 335174 (https://www.dell.com/support/kbdoc/335174) for details.') if isResults(&INFO); printResults(); } # End checkDriveFirmware $CHECKS{'checkFCOF022318EE'}{'Description'} = 'Checks drive firmware versions for the criteria of FCO F022318EE'; sub checkFCOF022318EE { return if wasCalled(); return unsupported('Unsupported on virtual platforms') if isVirtual(); printCheck('FCO F022318EE/KB517366'); gatherDriveFirmware(); if ($RPS{'Drive Firmware'}{'F022318EE'}) { warning(sprintf('There are %s drives(s) that are running firmware '. 'versions affected by F022318EE: %s', scalar(@{$RPS{'Drive Firmware'}{'F022318EE'}}), join(',', @{$RPS{'Drive Firmware'}{'F022318EE'}}))); warning('The identified drives should be updated, smartfailed, '. 'and then formatted.'); info('For additional details see KB517366 '. '[https://www.dell.com/support/kbdoc/517366 - Gen6: Ultrastar 7K6000 '. '(Aries-KP) 2TB and 4TB SED drives on firmware D930 may '. 'unexpectedly smartfail]'); } printResults(); } # End checkFCOF022318EE $CHECKS{'checkFCOF031617FC'}{'Description'} = 'Checks drive firmware versions for the criteria of KB 469133.'; sub checkFCOF031617FC { return if wasCalled(); return unsupported('Unsupported on virtual platforms') if isVirtual(); printCheck('FCO F031617FC/KB469133'); gatherDriveFirmware(); if ($RPS{'Drive Firmware'}{'KB469133'}) { warning(sprintf('There are %s drive(s) that are running firmware '. 'versions affected by FCO F031617FC/KB469133: %s ', scalar(@{$RPS{'Drive Firmware'}{'KB469133'}}), join(', ', @{$RPS{'Drive Firmware'}{'KB469133'}}))); warning('The identified drives should be smartfailed and replaced.'); info('For additional details see KB469133 '. '[https://www.dell.com/support/kbdoc/469133 - Upgrading Seagate '. 'harddrives from firmware SN04 or SN06 to a newer firmware '. 'version]'); } printResults(); } $CHECKS{'checkFCOF042415EE'}{'Description'} = 'Checks the cluster to see if it meets criteria for FCO F042415EE'; sub checkFCOF042415EE { return if wasCalled(); return unsupported('Unsupported on virtual platforms') if isVirtual(); printCheck('FCO F042415EE'); gatherDriveFirmware(); if ($RPS{'Drive Firmware'}{'F042415EE'}) { warning(sprintf('There are %s drive(s) that are running firmware '. 'versions affected by F042415EE: %s', scalar(@{$RPS{'Drive Firmware'}{'F042415EE'}}), join(',', @{$RPS{'Drive Firmware'}{'F042415EE'}}))); warning('The identified drives should be updated by Dell EMC.'); info('For additional details see KB472173 '. '[https://www.dell.com/support/kbdoc/472173 -F042415EE Isilon - 800GB '. 'SSD FW Upgrade - Installed Systems]'); } @{$RPS{'FCO F042415EE'}{'Nodes'}} = @{$RPS{'FCO'}{'F042415EE'}{'Nodes'}} if $RPS{'FCO'}{'F042415EE'}{'Nodes'}; printResults(); } # End checkFCOF042415EE $CHECKS{'checkKB535582'}{'Description'} = 'Checks if at risk for KB 535582'; sub checkKB535582 { if (wasCalled()) { return } checkDriveFirmware(); checkNodeFirmware(); checkPatches(); printCheck('KB 535582'); my $SPF_AFFECTED = '%s nodes may be affected by KB 535582 during a drive firmware update: %s'; my $MSG_REBOOT = 'Following drive firmware updates the drives may continue to report out-of-date in the firmware output. To show the correct firmware version in the firmware table, reboot the affected node(s). This reboot may be done as a standalone reboot or by any activity that reboots the nodes.'; my $MSG_ADDED_REBOOT = 'An additional reboot activity was added to accommodate for KB 535582.'; my $MSG_ACTIVITY = 'It is recommended to perform the subsequent reboot activity in the same maintenance window as the drive firmware update (KB 535582)'; my $SPF_ACTIVITY = 'Only the following nodes require rebooting: %s'; my $MSG_KB = 'For additional details review KB 535582 [https://www.dell.com/support/kbdoc/535582 - [Isilon] Certain drives may be smartfailed while updating drive firmware on Gen6 nodes]'; # Flag if KB535582 will require nodes to be rebooted following firmware updates if ($RPS{'Drive Firmware'}{'KB535582'}) { critical(sprintf($SPF_AFFECTED, scalar(@{$RPS{'Drive Firmware'}{'KB535582'}}), compressRange(@{$RPS{'Drive Firmware'}{'KB535582'}}))); addCustomerInformation('Upgrade Plan', critical($MSG_REBOOT)); my $nfp_reboot = 1; if (!$RPS{'Node Firmware'}{'Nodes'}) { $nf_reboot = 0; } else { my @nodes = array_duplicates(@{$RPS{'Drive Firmware'}{'KB535582'}}, @{$RPS{'Node Firmware'}{'Nodes'}}); if (scalar(@nodes) != scalar(@{$RPS{'Drive Firmware'}{'KB535582'}})) { $nfp_reboot = 0; } } # Check if there might be node reboots from other activities, if not add one if (!isDestinationOneFS() and !$RPS{'Pre Upgrade Patches'} and !$nfp_reboot and !$RPS{'Proactive Reboot'}) { addCustomerInformation('Upgrade Plan', $MSG_ADDED_REBOOT); push(@{$RPS{'Drive Support Package'}{'Additional Comments'}}, $MSG_ACTIVITY); $RPS{'Drive Firmware Reboot'}{'Position'} = '1'; @{$RPS{'Drive Firmware Reboot'}{'Nodes'}} = @{$RPS{'Drive Firmware'}{'KB535582'}}; push(@{$RPS{'Drive Firmware Reboot'}{'Additional Comments'}}, sprintf($SPF_ACTIVITY, compressRange(@{$RPS{'Drive Firmware'}{'KB535582'}}))); } } if (isResults(&INFO)) { info($MSG_KB) } printResults(); } # gatherCTO: Gathers CTO capable node information sub gatherCTO { if (wasCalled()) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'isi_inventory_tool --help|grep CTO']; my $CMD_LOGS = q[2>&1 grep -Hm5 . */asbuilt.xml]; my $RE_NOT_CTO = q[is only supported on CTO capable nodes]. q[|this node is not CTO capable]; # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # Store foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } my ($res) = grep {/$RE_NOT_CTO/} @{$gather->{$lnn}}; $NODE_INFO{$lnn}{'CTO'} = $res?0:1; debug("NODE_INFO->$lnn->CTO->$NODE_INFO{$lnn}{'CTO'}"); } } # End gatherCTO ### # gatherDriveFirmware: Gathers drive firmware information from each # node and stores it in $CLUSTER_INFO{'Drive Firmware'} sub gatherDriveFirmware { if (wasCalled()) { return } if (isVirtual()) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'isi_radish -q']; my $CMD_LOGS = q[2>&1 grep -H '^Bay' */isi_radish]; my $RE_DRIVE = qr/Bay\s+(\S+)\/\S+\s+is\s+(\S.*?)\s+FW:(\S+)\s+SN:/; # Gather my $gather = gatherCluster($RPS{'Live'}?$CMD_LIVE:$CMD_LOGS); # Compile my %fw = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { if ($line !~ $RE_DRIVE) { next } my ($bay, $model, $firmware) = ($1, $2, $3); # Compile %fw to contain: # Model -> Firmware -> Count/Nodes $fw{$model}{$firmware}{'Counts'}{$lnn}++; $fw{$model}{$firmware}{'Count'}++; @{$fw{$model}{$firmware}{'Nodes'}} = merge(@{$fw{$model}{$firmware}{'Nodes'}}, $lnn); # Firmware at risk for FCO F022318EE if ($model =~ /H5472602CLAR2000|H5472604CLAR4000/ and $firmware =~ /D930/) { push(@{$RPS{'Drive Firmware'}{'F022318EE'}}, "$lnn:bay$bay"); } # Firmware update may be at risk for KB 535582 if (($model =~ /ST8000NM0045-1RL112/ and $firmware =~ /UG05/) or ($model =~ /HUS726020ALN610/ and $firmware =~ /APGNT840|APGNTDCH/) or ($model =~ /HUS726040ALN610/ and $firmware =~ /APGNT840|APGNTDCH/)) { @{$RPS{'Drive Firmware'}{'KB535582'}} = merge(@{$RPS{'Drive Firmware'}{'KB535582'}}, $lnn); } # Firmware impacted by KB 502084 if (($model =~ /HITACHI HUSMM151CLAR1600/ and $firmware =~ /D344/) or ($model =~ /HGST HUSMM1680ASS205/ and $firmware =~ /D252|d322|D326/)) { @{$RPS{'Drive Firmware'}{'KB502084'}} = merge(@{$RPS{'Drive Firmware'}{'KB502084'}}, $lnn); } # Firmware impacted by KB 490849 if ($model =~ /ST.000NM00.3/ and $firmware =~ /^SN0[23]$|^SNG4$|^000[34]$/) { @{$RPS{'Drive Firmware'}{'KB490849'}} = merge(@{$RPS{'Drive Firmware'}{'KB490849'}}, $lnn); } # Firmware impacted by KB 469133 if ($firmware =~ /^SN0[46]$/) { push(@{$RPS{'Drive Firmware'}{'KB469133'}}, "$lnn:bay$bay"); } # Firmware impacted by KB 88930 if ($model =~ /HUA723030ALA640/ and $firmware =~ /MKAOA[35][8BV]0/) { @{$RPS{'Drive Firmware'}{'KB88930'}} = merge(@{$RPS{'Drive Firmware'}{'KB88930'}}, $lnn); } # Firmware impacted by FCO F042415EE if ($model =~ /HUSMM8080ASS205/ and $firmware eq 'D210') { push(@{$RPS{'Drive Firmware'}{'F042415EE'}}, "$lnn:bay$bay"); @{$RPS{'FCO'}{'F042415EE'}{'Nodes'}} = merge(@{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}, $lnn); } # Firmware impacted by ETA 193819 if ($model =~ /HUS7240[234]0ALA640/ and $firmware =~ /A[8AB][BCW]0/) { @{$RPS{'ETA'}{'193819'}{'Nodes'}} = merge(@{$RPS{'ETA'}{'193819'}{'Nodes'}}, $lnn); } } } # Save if (%fw) { debug(toJSON($CLUSTER_INFO{'Drive Firmware'} = \%fw)) } # Add FCO/ETA messages if ($RPS{'FCO'}{'F042415EE'}{'Nodes'}) { $RPS{'FCO'}{'F042415EE'}{'Severity'} = 'WARN'; $RPS{'FCO'}{'F042415EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}). ' nodes are affected by FCO F042415EE: '. compressRange(@{$RPS{'FCO'}{'F042415EE'}{'Nodes'}}); } if ($RPS{'ETA'}{'193819'}{'Nodes'}) { $RPS{'ETA'}{'193819'}{'Severity'} = 'INFO'; $RPS{'ETA'}{'193819'}{'Message'} = scalar(@{$RPS{'ETA'}{'193819'}{'Nodes'}}). ' nodes are affected by ETA 193819: '. compressRange(@{$RPS{'ETA'}{'193819'}{'Nodes'}}); } } # End gatherDriveFirmware $CHECKS{'checkDTA000194434'}{'Description'} = 'Checks for criteria of KB 000194434'; sub checkDTA000194434 { if (wasCalled()) { return } if (!isDestinationOneFS()) { return unsupported('Requires target_version parameter'); } if (!isDestinationOneFS(qr/^9\.2\.1\.[56]$|^9\.3\.0\.[012]$/)) { return unsupported('Supported when target_version is set to 9.2.1.5, 9.2.1.6, 9.3.0.0, 9.3.0.1, or 9.3.0.2'); } if (isCurrentOneFS('>=9.2.1.5')) { return unsupported('Supported for OneFS versions prior to 9.2.1.5'); } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen5'} and !$CLUSTER_INFO{'Gen6'}) { return unsupported('Supported on Generation 5 and Generation 6 hardware'); } # Setup printCheck('DTA 000194434'); my $SPF_SSD_SEDS = 'During an upgrade to OneFS %s, an issue exists that may lead to failures of the Self-Encrypting Solid State Drives on the following nodes: %s'; my $MSG_NEXT_ACTION_93 = 'This issue has been addressed in OneFS 9.3.0.3 and newer releases, consider instead updating to the latest available patch release for OneFS 9.3'; my $MSG_NEXT_ACTION_921 = 'This issue has been addressed in OneFS 9.2.1.7 and newer OneFS 9.2.1 releases, consider instead updating to the latest available patch release for OneFS 9.2.1'; my $MSG_KB = 'For details see KB 000194434 (https://www.dell.com/support/kbdoc/000194434)'; # Analyze my @ssd_sed_nodes = (); foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'Product'}) { missing($lnn); next; } if ($NODE_INFO{$lnn}{'Generation'} ne 'Gen5' and $NODE_INFO{$lnn}{'Generation'} ne 'Gen6') { next } if ($NODE_INFO{$lnn}{'Product'} =~ /SSD-SED/) { push(@ssd_sed_nodes, $lnn); } } # Flag if (@ssd_sed_nodes) { fail(sprintf($SPF_SSD_SEDS, isDestinationOneFS(), compressRange(@ssd_sed_nodes))); if (isDestinationOneFS('>=9.3')) { fail($MSG_NEXT_ACTION_93); } else { fail($MSG_NEXT_ACTION_921); } info($MSG_KB); } # Output printResults(); } # End checkDTA000194434 $CHECKS{'checkET004252'}{'Description'} = 'Checks for criteria of ET004252'; sub checkET004252 { if (wasCalled()) { return } if (!hasDellSwitch()) { return unsupported('No Dell Networking switches detected'); } # Setup printCheck('ET004252'); my $MSG_AT_RISK_MAJOR = 'The Dell Networking switches used for backend'. ' connectivity are running DNOS versions that are at risk for ET004252.'; my $MSG_REM = 'The Dell Networking switch updates are eligible for'. ' remote updates to 10.5.0.6C1'; my $MSG_NOT_REM = 'The Dell Networking switch updates are NOT eligible for'. ' remote updates to 10.5.0.6C1 and must be updated on-site'; my $MSG_NO_SWITCHES = 'Could not discover switch details.'. ' It may be necessary to update the list of recognized Dell switch'. ' OUIs on the Isilon cluster. For additional details see KB'. ' 000071455 [https://www.dell.com/support/kbdoc/000071455/onefs-8-1-2-and-8-2-x-dell-backend-switch-oui-missing-from-cluster-causing-onefs-not-to-recognize-fetch-switches-ipv6-address-and-other-management-features-from-the-switch]'; my $MSG_ODD_SWITCHES = 'An odd number of switches were found, there could'. ' be issues with one of the two backend networks preventing gathering'. ' of switch details'; my $MSG_INFO = 'For additional details see KB 000185548'. ' [https://www.dell.com/support/kbdoc/en-us/000185548/dell-switch-certificate-expiration-on-dell-switches-used-for-isilon-backend-ethernet]'; my $RE_DELL = qr/(Dell\s+\S+)\s+-\s+v(\d+\.\d+\.\S+)\s/; # Analyze my $count_switches = 0; my $count_updates = 0; my $remote_eligible = 0; if (defined $CLUSTER_INFO{'Switches'}) { foreach my $switches (values %{$CLUSTER_INFO{'Switches'}}) { foreach my $switch (@$switches) { debug("Switch: $switch"); $count_switches++; if ($switch !~ $RE_DELL) { next } my ($switch_type, $switch_os) = ($1, $2); debug("Switch Type: $switch_type, Switch OS: $switch_os"); if ($switch_os eq '10.5.0.6C1') { # DNOS update already performed next; } elsif ($switch_os eq '10.5.0.6') { # DNOS update remote eligible $count_updates++; $remote_eligible++; } elsif (compareVersions($switch_os, '10.5.0.6') < 0) { # DNOS update not remote eligible $count_updates++; } } } } # Flag if at risk and remote eligibility if ($count_updates) { critical($MSG_AT_RISK_MAJOR); # Flag if eligible or not for remote execution if ($count_updates == $remote_eligible) { # Eligble for remote execution info($MSG_REM); } else { # Not eligible for remote execution info($MSG_NOT_REM); } } # Flag issues in data if (!$count_switches) { # No switches? warning($MSG_NO_SWITCHES); } if ($count_switches > 0 and ($count_switches % 2) != 0) { # Odd number of switches reported warning($MSG_ODD_SWITCHES); } if (isResults(&WARN)) { info($MSG_INFO) } # Output printResults(); } # End checkET004252 ### # checkETAs: Checks for ETAs, version specific ETAs. $CHECKS{'checkETAs'}{'Description'} = 'Checks for Technical Advisories'; sub checkETAs { return if wasCalled(); checkNodeFirmware(); checkKB533516(); checkDTA000194434(); # checkET004252(); printCheck("ETAs"); gatherDriveFirmware(); gatherDTA194319(); gatherDTA540272(); gatherDTA533516(); gatherDTA526050(); gatherDTA525928(); gatherETA519384(); gatherETA518924(); gatherETA491747(); gatherETA483840(); gatherETA209918(); gatherETA207785(); gatherETA205085(); gatherETA202452(); gatherETA198924(); gatherETA194692(); gatherETA180317(); # ETA 208773 if (isCurrentOneFS(qr/^7\.1\.1\.[67]$|^7\.2\.0\.[34]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'208773'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208773'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 208773"); } if (isDestinationOneFS(qr/^7\.1\.1\.[67]$|^7\.2\.0\.[34]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'208773'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208773'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 208773"); } # ETA 208772 if (isCurrentOneFS(qr/^7\.2\.1\.0$|^7\.2\.0\.[01234]$|^7\.1\.1\.[01234567]$|^7\.1\.0\.[0123456]$/)) { $RPS{'ETA'}{'208772'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208772'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 208772"); } if (isDestinationOneFS(qr/^7\.2\.1\.0$|^7\.2\.0\.[01234]$|^7\.1\.1\.[01234567]$|^7\.1\.0\.[0123456]$/)) { $RPS{'ETA'}{'208772'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'208772'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 208772"); } # ETA 207782 if (isCurrentOneFS(qr/^7\.2\.0\.[012]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'207782'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207782'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 207782"); } if (isDestinationOneFS(qr/^7\.2\.0\.[012]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'207782'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207782'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 207782"); } # ETA 204898 if (isCurrentOneFS(qr/^7\.2\.0\.[0123]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'204898'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'204898'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 204898"); } if (isDestinationOneFS(qr/^7\.2\.0\.[0123]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'204898'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'204898'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 204898"); } # ETA 203815 if (isCurrentOneFS(qr/^7\.1\.1\.[012345]$|^7\.2\.0\.[0123]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'203815'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'203815'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 203815"); } if (isDestinationOneFS(qr/^7\.1\.1\.[012345]$|^7\.2\.0\.[012]$|^7\.2\.1\.0$/)) { $RPS{'ETA'}{'203815'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'203815'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 203815"); } # ETA 200097 if (isCurrentOneFS(qr/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'200097'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200097'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 200097"); } if (isDestinationOneFS(qr/^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'200097'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200097'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 200097"); } # ETA 200096 if (isCurrentOneFS(qr/^7\.1\.1\.[345]$|^7\.2\.0\.[12]$|^7\.2\.1\.1$/)) { $RPS{'ETA'}{'200096'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200096'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 200096"); } if (isDestinationOneFS(qr/^7\.1\.1\.[345]$|^7\.2\.0\.[12]$|^7\.2\.1\.1$/)) { $RPS{'ETA'}{'200096'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'200096'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 200096"); } # ETA 199379 if (isCurrentOneFS(qr/^6|^7\.0|^7\.1\.0|^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'199379'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'199379'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 199379"); } if (isDestinationOneFS(qr/^6|^7\.0|^7\.1\.0|^7\.1\.1\.[0123]$|^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'199379'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'199379'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 199379"); } # ETA 198187 if (isCurrentOneFS('7.2.0.0')) { $RPS{'ETA'}{'198187'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198187'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 198187"); } if (isDestinationOneFS('7.2.0.0')) { $RPS{'ETA'}{'198187'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198187'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 198187"); } # ETA 197460 if (isCurrentOneFS('7.2.0.0')) { $RPS{'ETA'}{'197460'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'197460'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 197460"); } if (isDestinationOneFS('7.2.0.0')) { $RPS{'ETA'}{'197460'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'197460'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 197460"); } # ETA 190094 if (isCurrentOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'190094'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'190094'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 190094"); } if (isDestinationOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'190094'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'190094'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 190094"); } # ETA 189234 if (isCurrentOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'189234'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'189234'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 189234"); } if (isDestinationOneFS(qr/^7\.1\.0\.[0123]|^7\.1\.1\.0$/)) { $RPS{'ETA'}{'189234'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'189234'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 189234"); } # ETA 188835 if (isCurrentOneFS('7.1.0.3')) { $RPS{'ETA'}{'188835'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'188835'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 188835"); } if (isDestinationOneFS('7.1.0.3')) { $RPS{'ETA'}{'188835'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'188835'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 188835"); } # ETA 185153 if (isCurrentOneFS(qr/^6\.5\.5\.[01][0123456789]|^6\.5\.5\.2[01]$|^7\.0\.0\.0$|^7\.0\.1\.[012345]$/)) { $RPS{'ETA'}{'185153'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'185153'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 185153"); } if (isDestinationOneFS(qr/^6\.5\.5\.[01][0123456789]|^6\.5\.5\.2[01]$|^7\.0\.0\.0$|^7\.0\.1\.[012345]$/)) { $RPS{'ETA'}{'185153'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'185153'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 185153"); } if ($RPS{'ETA'}) { foreach my $eta (sort {$a <=> $b} keys %{$RPS{'ETA'}}) { next unless $RPS{'ETA'}{$eta}{'Message'} and $RPS{'ETA'}{$eta}{'Severity'}; info(@{$RPS{'ETA'}{$eta}{'Message'}}) if $RPS{'ETA'}{$eta}{'Severity'} eq "INFO"; warning(@{$RPS{'ETA'}{$eta}{'Message'}}) if $RPS{'ETA'}{$eta}{'Severity'} eq "WARN"; fail(@{$RPS{'ETA'}{$eta}{'Message'}}) if $RPS{'ETA'}{$eta}{'Severity'} eq "FAIL"; } } printResults(); } # End checkETAs $CHECKS{'checkKB533516'}{'Description'} = "Checks if cluster uses an IP for AWS CloudPool accounts putting it at risk for DTA 533516"; sub checkKB533516 { return if wasCalled(); return unsupported("No active CloudPools license on this cluster.") unless hasLicense("CloudPools"); return unsupported("Does not apply to prior to OneFS 8") if isCurrentOneFS('<8'); return unsupported("Does not apply to OneFS 8.2 and later") if isCurrentOneFS('>=8.2'); printCheck("KB533516"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig -t cpool_cfg]. q[ | grep -E '\.uri |\.account_name |\.ptype ']); } else { $gather = gatherLocal(qq[2>&1 sqlite3 local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/Gconfig.cpool_config.sqlite 'select * from kv_table']. q[ | grep -E '\.uri |\.account_name |\.ptype ']); } # Analyze information for issues. my %accounts = (); my $re_line = qr[^account\.(\d+)\.(account_name|ptype|uri) .*= "?(.*?)"?$]; foreach my $line (@$gather) { $accounts{$1}{$2} = $3 if $line =~ $re_line; } my @accounts_with_ip = (); foreach my $id (keys %accounts) { if ($accounts{$id}{'uri'} and $accounts{$id}{'uri'} =~ qr[//\d+\.\d+\.\d+\.\d+] and $accounts{$id}{'ptype'} and $accounts{$id}{'ptype'} =~ qr[CPT_AWS] and $accounts{$id}{'account_name'}) { push(@accounts_with_ip, $accounts{$id}{'account_name'}); } } if (@accounts_with_ip) { critical("The following CloudPool accounts are configured for AWS using an IP for the URI: ".join(",", @accounts_with_ip)); critical(" For V4 signatures a FQDN is required."); info("For additional details see KB533516 [https://www.dell.com/support/kbdoc/533516 - DTA 533516: Isilon OneFS: Amazon is ending support for Authentication V2 which may prevent CloudPools from replicating data to Amazon AWS instance]."); } # Output Check result printResults(); } # End checkKB533516 ### # gatherDTA194319: Checks to see if the cluster meets criteria for DTA 194319 sub gatherDTA194319 { if (wasCalled()) { return } if (isCurrentOneFS(qr/^(?:8\.2|9\.0|9\.1\.1|9\.2\.0)\./) or isCurrentOneFS(qr/^9\.1\.0\.(?:1[0123]|[0123456789])$/) or isCurrentOneFS(qr/^9\.2\.1\.[0123456]$/)) { $RPS{'ETA'}{'194319'}{'Severity'} = 'INFO'; push(@{$RPS{'ETA'}{'194319'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by DTA 194319"); } if (isDestinationOneFS(qr/^(?:8\.2|9\.0|9\.1\.1|9\.2\.0)\./) or isDestinationOneFS(qr/^9\.1\.0\.(?:1[0123]|[0123456789])$/) or isDestinationOneFS(qr/^9\.2\.1\.[0123456]$/)) { $RPS{'ETA'}{'194319'}{'Severity'} = 'INFO'; push(@{$RPS{'ETA'}{'194319'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by DTA 194319"); } } # End gatherDTA194319 ### # gatherDTA525928: Checks to see if the cluster meets criteria for DTA 525928. sub gatherDTA525928 { return if wasCalled(); if (isCurrentOneFS(qr/^(8.0.0.7|8.1.0.4)$/)) { $RPS{'ETA'}{'525928'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'525928'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by DTA 525928."); } if (isDestinationOneFS(qr/^(8.0.0.7|8.1.0.4)$/)) { $RPS{'ETA'}{'525928'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'525928'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by DTA 525928."); } } # End gatherDTA525928 ### # gatherDTA526050: Checks to see if the cluster meets criteria for ETA 526050. sub gatherDTA526050 { return if wasCalled(); if (isCurrentOneFS(qr/^7\.[12]/)) { $RPS{'ETA'}{'526050'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'526050'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by DTA 526050"); } if (isDestinationOneFS(qr/^7\.[12]/)) { $RPS{'ETA'}{'526050'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'526050'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by DTA 526050"); } } # End gatherDTA526050 sub gatherDTA533516 { return if wasCalled(); if (isCurrentOneFS(qr/^8\.[01]/)) { $RPS{'ETA'}{'533516'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'533516'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by DTA 533516."); } if (isDestinationOneFS(qr/^8\.[01]/)) { $RPS{'ETA'}{'533516'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'533516'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by DTA 533516."); } } ### # gatherDTA540272: Checks to see if the cluster meets criteria for DTA 540272 sub gatherDTA540272 { if (wasCalled()) { return } if (isCurrentOneFS('>=9')) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'sysctl efs.bam.rename_event_coherency']; my $CMD_LOGS = q[2>&1 grep -H efs.bam.rename_event_coherency */etc.tar/etc/sysctl.conf]; my $RE_SYSCTL = qr/efs.bam.rename_event_coherency(?:=|: )(\d+)/; # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); my $sysctl_value = -1; foreach my $lnn (@LNNS) { if (!$gather{$lnn}) { next } foreach my $line (@{$gather{$lnn}}) { if ($line =~ $RE_SYSCTL) { if ($1 > $sysctl_value) { $sysctl_value = $1 } } } } if ($sysctl_value == 1) { $RPS{'ETA'}{'540272'}{'Severity'} = 'INFO'; push(@{$RPS{'ETA'}{'540272'}{'Message'}}, 'Cluster at risk for DTA 540272'); } } # End gatherDTA540272 ### # gatherETA180317: Gathers whether the cluster is running versions affected by ETA 180317 sub gatherETA180317 { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'grep -m1 "QLogic 12[38]00" /var/log/opensm-[12].topo']); } else { $gather = gatherCluster(qq[2>&1 grep -m1 -H "QLogic 12[38]00" */varlog.tar/log/opensm-[12].topo]); } # Analyze information for issues. my $meets_eta180317 = 0; foreach my $lnn (@LNNS) { next unless ($gather->{$lnn}); last if $meets_eta180317; foreach my $line (@{$gather->{$lnn}}) { last if $meets_eta180317; $meets_eta180317 = 1 if $line =~ /QLogic 12[38]00/; } } if ($meets_eta180317) { $RPS{'ETA'}{'180317'}{'Severity'} = "WARN"; push(@{$RPS{'ETA'}{'180317'}{'Message'}}, "InfiniBand switch firmware may be running a version affected by ETA 180317"); } } # End gatherETA180317 ### # gatherETA194692: Gathers whether the cluster is running versions affected by ETA 194692 sub gatherETA194692 { return if wasCalled(); # ETA 194692 my $regex = qr/^6\.5\.5\.\d$|^6\.5\.5\.1\d$|^6\.5\.5\.2[012]$|^7\.0\.2\.[01]$|^7\.1\.0\.[012345]$|^7\.1\.1\.1$|^7\.2\.0\.0$/; if (isCurrentOneFS($regex)) { $RPS{'ETA'}{'194692'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'194692'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 194692"); } if (isDestinationOneFS($regex)) { $RPS{'ETA'}{'194692'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'194692'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 194692"); } } # End gatherETA194692 ### # gatherETA198924: Gathers whether the cluster is running versions affected by ETA 198924 sub gatherETA198924 { return if wasCalled(); gatherProductInfo(); # ETA 198924 my $has_hd400_nodes = 0; foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Product'}; $has_hd400_nodes = 1 if $NODE_INFO{$lnn}{'Product'} =~ /^HD400/; } if ($has_hd400_nodes) { if (isCurrentOneFS(qr/^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'198924'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198924'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 198924"); } if (isDestinationOneFS(qr/^7\.2\.0\.[01]$/)) { $RPS{'ETA'}{'198924'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'198924'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 198924"); } } } # End gatherETA198924 ### # gatherETA202452: Gathers whether the cluster is running versions affected by ETA 202452 sub gatherETA202452 { return if wasCalled(); # ETA 202452 my $regex = qr/^7\.1\.0\.[0123456]$|^7\.1\.1\.[0123456]$|^7\.2\.0\.[0123]$|^7\.2\.1\.0$/; if (isCurrentOneFS($regex)) { $RPS{'ETA'}{'202452'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'202452'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 202452"); } if (isDestinationOneFS($regex)) { $RPS{'ETA'}{'202452'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'202452'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 202452"); } } # End gatherETA202452 ### # gatherETA205085: checks for conditions for ETA 205085. # ETA 205085: Isilon OneFS: Legacy NFSv3 clients that require 32-bit # cookies may encounter data unavailability in OneFS 7.2.1.0 and 7.2.0.0 - 7.2.0.2 sub gatherETA205085 { if (wasCalled()) { return } if (isCurrentOneFS('>=7.2') or isDestinationOneFS('<7.2')) { return } # ETA 205085 $RPS{'ETA'}{'205085'}{'Severity'} = "WARN"; push(@{$RPS{'ETA'}{'205085'}{'Message'}}, "Upgrade from OneFS $CLUSTER_INFO{'OneFS'} to OneFS $RPS{'Target'} affected by ETA 205085"); addCustomerInformation ("ETA 205085", "NFSv3 clients that require 32-bit cookies may be unable to access data or view files that are contained in an NFS export due to a change in the way OneFS returns read directory (READDIR) cookies. This situation may result in data unavailability.", "If you have NFSv3 clients that require 32-bit cookies, review ETA 205085 (https://www.dell.com/support/kbdoc/301863) and take any appropriate action prior to the upgrade."); } # End gatherETA205085 ### # gatherETA207785: Gathers whether the cluster is running versions affected by ETA 207785 sub gatherETA207785 { return if wasCalled(); # ETA 207785 if (isCurrentOneFS(qr/^7\.1\.1\.6$/)) { $RPS{'ETA'}{'207785'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207785'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 207785"); } if (isDestinationOneFS(qr/^7\.1\.1\.6$/)) { $RPS{'ETA'}{'207785'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'207785'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 207785"); } } # End gatherETA207785 ### # gatherETA209918: Gathers whether the cluster is running versions affected by ETA 209918 sub gatherETA209918 { return if wasCalled(); # ETA 209918 my $regex = qr/^7\.1\.0\.[0123456]$|^7\.1\.1\.[012345]$|^7\.2\.0\.[012]$|^7\.2\.1\.0$/; if (isCurrentOneFS($regex)) { $RPS{'ETA'}{'209918'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'209918'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 209918"); } if (isDestinationOneFS($regex)) { $RPS{'ETA'}{'209918'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'209918'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 209918"); } } # End gatherETA209918 ### # gatherETA483840: checks for conditions for ETA 483840. # ETA 483840: Isilon OneFS: Isilon OneFS: Non-UTF-8 encoded files or # directories may be inaccessible to NFSv3 and NFSv4 clients after # upgrading OneFS clusters from version 7.1.x or earlier to 7.2.0.0 or later sub gatherETA483840 { if (wasCalled()) { return } # Must be for a OneFS upgrade if (!isDestinationOneFS()) { return } # Does not apply if already at OneFS 7.2 or higher return if isCurrentOneFS('>=7.2'); # Does not apply if upgrade is not to OneFS 7.2 or higher if (isDestinationOneFS('<7.2')) { return } # Does not apply if upgrade is to OneFS 8.1.1 or higher if (isDestinationOneFS('>=8.1.1')) { return } # ETA 483840 $RPS{'ETA'}{'483840'}{'Severity'} = "WARN"; push(@{$RPS{'ETA'}{'483840'}{'Message'}}, "Upgrade from OneFS $CLUSTER_INFO{'OneFS'} to OneFS $RPS{'Target'} affected by ETA 483840"); addCustomerInformation("ETA 483840", "OneFS may inaccurately translate non-UTF-8 encoded file and directory names that are accessed by NFS clients on a small percentage of clusters that have been upgraded from OneFS 7.1.x or earlier to OneFS 7.2.0.0 or later.", "If you have NFS clients, review ETA 483840 (https://www.dell.com/support/kbdoc/483840) and complete steps in the resolution to determine if this may put access to your data at risk."); } # End gatherETA483840 ### # gatherETA491747: Checks to see if the cluster meets criteria for ETA 491747. sub gatherETA491747 { if (wasCalled()) { return } if (isCurrentOneFS('>=7.2.1.4') or isCurrentOneFS('7.1.1.11')) { return } gatherProductInfo(); gatherNodeFirmware(); my @ETA491747 = (); foreach my $lnn (@LNNS) { if ($NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(A100|X410|X210|S210|HD400|NL410)/ and $NODE_INFO{$lnn}{'Firmware'}{'Mellanox'}) { push(@ETA491747, $lnn); } } if (@ETA491747) { if (isCurrentOneFS(qr/^7\.2\.1\.[0123]$/) || isCurrentOneFS(qr/^7\.1\.1\.([0123456789]|10)$/)) { $RPS{'ETA'}{'491747'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'491747'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 491747 on ".scalar(@ETA491747)." nodes: ".compressRange(@ETA491747)); } if (isDestinationOneFS(qr/^7\.2\.1\.[0123]$/) || isDestinationOneFS(qr/^7\.1\.1\.([0123456789]|10)$/)) { $RPS{'ETA'}{'491747'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'491747'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 491747 on ".scalar(@ETA491747)." nodes: ".compressRange(@ETA491747)); } } } # End gatherETA491747 ### # gatherETA518924: Checks to see if the cluster meets criteria for ETA 518924 sub gatherETA518924 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.1.1')) { return } # Setup my @ERRORS = ('bam_mark_blocks', 'dfm_mark_directory', 'ifm_mark_metatree', 'qdb_mark_inner', 'sbt_mark_tree', 'stf_mark_metatree'); my $GREP_ERRORS = join('|', @ERRORS); my $GREP_IGNORE = 'sudo: '; my $CMD_LOGS = qq[grep -a -h -E "$GREP_ERRORS" */varlog.tar/log/messages]. qq[ | grep -v "$GREP_IGNORE" | sort | tail -1]; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array -Q]. qq[ 'grep -a -h -E "$GREP_ERRORS" /var/log/messages']. qq[ | grep -v "$GREP_IGNORE" | sort | tail -1]; my $RE_LNN = '^\s*(\S+)\s+\S+\s+\S+-(\d+)'; my $SPF_LOCK = 'ETA 518924: Locking contention seen in the last 30 days, '. ' most recent occurrence seen on node %s at %s'; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $epoch = 0; my $lnn = 0; foreach (@$gather) { if (!/$RE_LNN/) { next } $epoch = parseTime($1); $lnn = $2; if ($epoch < $RPS{'Timestamp'} - 30 * 24 * 60 * 60) { $epoch = 0; } } # Flag $RPS{'ETA'}{'518924'}{'Severity'} = 'INFO'; push(@{$RPS{'ETA'}{'518924'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 518924"); if (isDestinationOneFS('<8.1.1')) { $RPS{'ETA'}{'518924'}{'Severity'} = 'INFO'; push(@{$RPS{'ETA'}{'518924'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 518924"); } if ($epoch and $lnn) { $RPS{'ETA'}{'518924'}{'Severity'} = 'WARN'; push(@{$RPS{'ETA'}{'518924'}{'Message'}}, sprintf($SPF_LOCK, $lnn, scalar(localtime($epoch)))); } } # End gatherETA518924 sub gatherETA519384 { return if wasCalled(); if (isCurrentOneFS('<8.0.0.6') or isCurrentOneFS(qr/^(8\.0\.1\.[01]|8\.1\.0\.0)$/)) { $RPS{'ETA'}{'519384'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'519384'}{'Message'}}, "Current OneFS $CLUSTER_INFO{'OneFS'} affected by ETA 519384"); } if (isDestinationOneFS('<8.0.0.6') or isDestinationOneFS(qr/^(8\.0\.1\.[01]|8\.1\.0\.0)$/)) { $RPS{'ETA'}{'519384'}{'Severity'} = "INFO"; push(@{$RPS{'ETA'}{'519384'}{'Message'}}, "Destination OneFS $RPS{'Target'} affected by ETA 519384"); } } # End gatherETA519384 $CHECKS{'checkBootDisks'}{'Description'} = 'Checks boot disk wear life'. ' remaining, firmware level, and historic error count'; sub checkBootDisks { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen5'} and !$CLUSTER_INFO{'Gen4'}) { return unsupported('Only supported on the following node types: S200,'. ' S210, X200, X210, X400, X410, NL400, NL410, and'. ' HD400'); } # Setup printCheck('Boot Disks'); my $CMD_LOGS = q[2>&1 grep -H . */isi_radish_bootdisk]; my $CMD_LIVEOLD = qq[2>&1 ${SUDO} isi_for_array 'isi_radish -a]. q[ "Internal J3" "Internal J4" "Carrier board J3" "Carrier board J4"']; my $CMD_LIVENEW = qq[2>&1 ${SUDO} isi_for_array 'isi_radish -a ada0 ada1']; my $RE_BD = qr/^\s*(?:Internal|Carrier board)?\s*(\S+\/\S+)\s+is\s+(.*)$/i; my $RE_LIFE = qr/(?:Percent Life Remaining|Percent Lifetime Left).*raw (\S+)\)/i; my $RE_ERROR = qr/error index\s+\d+\s+historic error count\s+(\d+)$/i; my $SPF_LIFE = 'Boot disk %s on node %s has %s life remaining'; my $SPF_ERROR = 'Boot disk %s on node %s has a historic error count of %s'; my $SPF_MASTER = 'Master boot disk information not found on the following nodes: %s'; my $SPF_SLAVE = 'Slave boot disk information not found on the following nodes: %s'; my $SPF_FW = 'The following boot devices are running firmware version'. ' 7.02k which is impacted by KB 489143: %s'; my $MSG_KB = 'For details review KB 335101'. ' [https://www.dell.com/support/kbdoc/000045857]'; # Gather my $gather = (); if (isLive()) { if (isCurrentOneFS('<8.1')) { $gather = gatherCluster($CMD_LIVEOLD); } else { $gather = gatherCluster($CMD_LIVENEW); } } else { $gather = gatherCluster($CMD_LOGS); } # Analyze my @fw_702k = (); my @no_master = (); my @no_slave = (); foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'Generation'} or ($NODE_INFO{$lnn}{'Generation'} ne 'Gen5' and $NODE_INFO{$lnn}{'Generation'} ne 'Gen4')) { next; } verboseHeader("$NODE_INFO{$lnn}{'Name'} - Boot Disks"); if (!$gather->{$lnn}) { missing($lnn); verbose('Missing'); next; } my $bd_location = ''; my $bd_details = ''; my $has_master = 0; my $has_slave = 0; foreach my $line (@{$gather->{$lnn}}) { if ($line =~ $RE_BD) { ($bd_location, $bd_details) = ($1, $2); verbose("Boot Device $bd_location:"); if ($bd_location =~ /ada0|ad0|ad2|ad4|ad6/) { $has_master = 1; verbose(' (Master device)'); } if ($bd_location =~ /ada1|ad1|ad3|ad5|ad7/) { $has_slave = 1; verbose(' (Slave device)'); } if ($bd_details =~ /^(.*?) FW:/) { verbose(" Model: $1"); } if ($bd_details =~ /SN:([^,]+)/) { verbose(" Serial Number: $1"); } if ($bd_details =~ /FW:(.*?) SN/) { verbose(" Firmware: $1"); } if ($bd_details =~ /FW:.*7\.02k/) { push(@fw_702k, "Node $lnn $bd_location"); } } if ($line =~ $RE_LIFE) { my $life = hex $1; verbose(" Wear Life Remaining: $life\%"); if ($life <= 5) { fail(sprintf($SPF_LIFE, $bd_location, $lnn, "$life%")); } elsif ($life <= 10) { warning(sprintf($SPF_LIFE, $bd_location, $lnn, "$life%")); } } if ($line =~ $RE_ERROR) { verbose(" Historic Error Count: $1"); if ($1 ne '0000') { warning(sprintf($SPF_ERROR, $bd_location, $lnn, $1)); } } } if (!$has_master) { push(@no_master, $lnn) } if (!$has_slave) { push(@no_slave, $lnn) } } # Flag if (@fw_702k) { warning(sprintf($SPF_FW, join(', ', @fw_702k))) } if (@no_master) { fail(sprintf($SPF_MASTER, compressRange(@no_master))) } if (@no_slave) { fail(sprintf($SPF_SLAVE, compressRange(@no_slave))) } if (isResults(&WARN)) { info($MSG_KB) } # Output printResults(); } # End checkBootDisks $CHECKS{'checkDriveLoad'}{'Description'} = "Checks the current load on the drives"; sub checkDriveLoad { if (wasCalled()) { return } printCheck("Drive Load"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { if (isCurrentOneFS('<8.0')) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi statistics drive --degraded]); } else { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi statistics drive --degraded --long]); } } else { $gather = gatherCluster(qq[2>&1 grep -H . */isi_statistics_drive]); } # Analyze information for issues. my ($bay, $typ, $tiq, $que, $max); if (isCurrentOneFS('<8.0')) { $bay = 0; $typ = 1; $tiq = 7; $que = 8; $max = 9; } else { $bay = 1; $typ = 2; $tiq = 11; $que = 12; $max = 15; } my @devices_highload = (); foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Drive Load"); verbose(sprintf("%-10s %-10s %-15s %-10s", "Location", "Type", "Time-in-queue", "Queue")); verbose(sprintf("%-10s %-10s %-15s %-10s", "-"x10, "-"x10, "-"x15, "-"x10)); if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { $line =~ s/^\s+|\s+$//g; next if $line =~ /^(LNN|Drive|Timestamp|-|Total)/; my @device = split(/\s+/, $line); next unless @device and $#device == $max; $device[$tiq] =~ s/ms//; verbose(sprintf("%-10s %-10s %-15s %-10s", $device[$bay], $device[$typ], $device[$tiq], $device[$que])); if (($device[$typ] eq 'SAS' and ($device[$tiq] >= 80 or $device[$que] >= 10)) or ($device[$typ] eq 'SATA' and ($device[$tiq] >= 100 or $device[$que] >= 5)) or ($device[$typ] eq 'SSD' and ($device[$tiq] >= 5 or $device[$que] >= 1))) { push(@devices_highload, $device[$bay]); } } } warning("High disk load detected on the following devices: ".join(",", @devices_highload)) if @devices_highload; # Output Check result printResults(); } # End checkDriveLoad $CHECKS{'checkDrivesHealth'}{'Description'} = "Checks health of drives and the drive stall timeout setting in sysctl"; sub checkDrivesHealth { if (wasCalled()) { return } printCheck("Drives Health"); gatherProductInfo(); # Gather device information my $gather = (); if (isCurrentOneFS('<8.0') and $RPS{'Live'}) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi devices|grep -v HEALTHY"); } elsif (isCurrentOneFS('<8.0')) { $gather = gatherCluster("2>&1 grep -H -v HEALTHY */isi_devices"); } elsif ($RPS{'Live'}) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array 'isi devices drive list|grep -v HEALTHY'"); } else { $gather = gatherCluster("2>&1 grep -H -v HEALTHY */isi_devices_drive_list"); } verboseHeader("Drives Health - Drives in a HEALTHY state are not reported"); # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Drives Health"); verbose(sprintf("%-10s %-10s", "Location", "State")); verbose(sprintf("%-10s %-10s", "-"x10, "-"x10)); if (!$gather->{$lnn}) { missing($lnn); next; } my $unpack_template = q[]; my $min_length = q[]; foreach my $line (@{$gather->{$lnn}}) { next if ($line =~ /^[\s-]*$/); next if ($line =~ /^\s*Total/); # Strip leading whitespace $line =~ s/^\s*//; my ($bay, $lnum, $state) = ""; my @unpacked_line = (); if ($line =~ /^\s*Lnn/) { $unpack_template = generateUnpackTemplate($line, "Lnn", "Location", "Device", "Lnum", "State", "Serial"); $min_length = length($line); next; } elsif ($unpack_template) { eval { $line = sprintf("%-${min_length}s", $line); @unpacked_line = unpack($unpack_template, $line); $bay = $1 if ($unpacked_line[1] =~ /Bay\s+(\S+)/); $lnum = $1 if ($unpacked_line[3] =~ /(\d+)/); $state = $1 if ($unpacked_line[4] =~ /(\S+)/); }; } else { $bay = $1 if ($line =~ /Bay\s+(\S+)/); $lnum = $1 if ($line =~ /Lnum\s+(\S+)/); $state = $1 if ($line =~ /\[(.*)\]/); } next unless ($lnum or $bay) and $state; my $identifier = ($bay?"Bay $bay":"Lnum $lnum"); verbose(sprintf("%-10s %-10s", $identifier, $state)); # Flag nodes if the SSDs are L3, this will be used for drive firmware and FCO activity scheduling if ($state =~ /L3/) { $NODE_INFO{$lnn}{'L3'} = 1; debug("NODE_INFO->${lnn}->L3=1"); } # Healthy states next if ($state =~ /HEALTHY|L3|BOOT_DRIVE|JOURNAL|EMPTY/); # Not healthy, but the upgrade can proceed if ($state =~ /STALLED|REPLACE|EMPTY|ERASE|USED/) { warning("Node $lnn $identifier is $state"); } # Unexpected state else { fail("Node $lnn $identifier is $state"); } } } info("Refer to KB 335105 (https://www.dell.com/support/kbdoc/335105) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkDrivesHealth $CHECKS{'checkDriveStallTimeout'}{'Description'} = "Checks current Drive Stall Timeout setting, recommend value is 3.5 seconds (3500000 microseconds) or higher"; sub checkDriveStallTimeout { return if wasCalled(); printCheck("Drive Stall Timeout"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array /sbin/sysctl hw.disk_event.thresh.slowacc_usec]); } else { $gather = gatherCluster(qq[2>&1 grep -H hw.disk_event.thresh.slowacc_usec */hw.disk_event]); } # Analyze information for issues. my $min_recommended_threshold = 3500000; my @nodes_less_than_threshold = (); verboseHeader("Drive Stall Timeout"); verbose(sprintf("%-10s %-10s", "Node", "Drive Stall Timeout Setting (microseconds)")); verbose(sprintf("%-10s %-10s", "-"x10, "-"x10)); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } my $threshold = 0; foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /hw.disk_event.thresh.slowacc_usec: (\d+)$/; $threshold = $1; verbose(sprintf("%-10s %-10s", $lnn, $threshold)); } if ($threshold and $threshold < $min_recommended_threshold) { push(@nodes_less_than_threshold, $lnn); } } if (@nodes_less_than_threshold) { warning("Drive stall timeout is not set to 3.5 seconds on the following nodes: ".compressRange(@nodes_less_than_threshold), "For details on increasing the drive stall timeout, see KB487685: https://www.dell.com/support/kbdoc/487685 - Increasing the drive stall timeout values in OneFS"); } printResults(); } # End checkDriveStallTimeout sub checkExpectedMemory { return if wasCalled(); checkHardwareStatus(); verboseHeader("checkExpectedMemory"); verbose(sprintf("%5s %10s %10s", "LNN", "Actual", "Expected")); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'RAM'}; # Current amount of RAM reported in isi_hw_status my $mem_current_int = $NODE_INFO{$lnn}{'RAM'}; my $mem_current_str = convertFromNumeric($mem_current_int, "G"); # Expected amount of RAM from the product info my @product_info = split(/-/, $NODE_INFO{$lnn}{'Product'}); # This won't work for older/IQ nodes next unless scalar(@product_info) > 3; my $mem_expected_str = $product_info[3]; my $mem_expected_int = convertToNumeric($mem_expected_str); # If the expected and current memory are more than 1GB off if (abs($mem_expected_int - $mem_current_int) > 1073741824) { fail("Memory mismatch node $lnn, expected $mem_expected_str, found $mem_current_str."); } # Verbose output verbose(sprintf("%5s %10s %10s", $lnn, $mem_current_str, $mem_expected_str)); } } # End checkExpectedMemory $CHECKS{'checkHardwareStatus'}{'Description'} = 'Checks battery health, power supplies, and gathers hardware details for use elsewhere.'; sub checkHardwareStatus { return if wasCalled(); printCheck('Hardware Status'); # Gather information my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'isi_hw_status -Lcisfvtbg|grep -v test']); } else { $gather = gatherCluster(q[2>&1 grep -H -v test */isi_hw_status]); } # Analyze information my $kb519119 = 0; foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - Hardware Status"); if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { verbose($line); if ($line =~ /Write to lm96000 fan2 configuration register failed/) { fail("Node $lnn is reporting '${line}'"); } elsif ($line =~ /hal_get_power_supply_type/) { fail("Node $lnn is reporting '${line}'"); } elsif ($line =~ /^\s*Product: (.*)$/) { $NODE_INFO{$lnn}{'Product'} = $1; debug("NODE_INFO->${lnn}->Product=$1"); } elsif ($line =~ /^\s*PROC: (.*)$/) { $NODE_INFO{$lnn}{'Cores'} = parseCoreCount($1); debug("NODE_INFO->${lnn}->Cores=$NODE_INFO{$lnn}{'Cores'}"); } elsif ($line =~ /^\s*PwrSupl: (\S+)/) { my $location = $1; my $firmware = ""; my $type = ""; $firmware = $1 if $line =~ /fw=(\S+)/; $firmware =~ s/\)//; $type = $1 if $line =~ /type=(\S+)/; $type =~ s/,//; if ($firmware ne "N/S") { $NODE_INFO{$lnn}{'Firmware'}{$location} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${location}=$firmware"); } if ($NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(A200|A2000|H400|H500|H600|F800)/ and $type !~ /^ARTESYN/) { $kb519119 = 1; } } elsif ($line =~ /^\s*Power Supply\s+(\S+)\s+(.*)$/) { next if $line =~ /=/; if ($2 ne "good") { fail("Power Supply $1 on node $lnn is $2"); } } elsif ($line =~ /^\s*Battery1:\s*\((\S*)\)\s*Battery2:\s*\((\S*)\)\s*$/) { if ($1 ne "okay") { fail("Battery 1 on node $lnn is $1"); } if ($2 ne "okay") { fail("Battery 2 on node $lnn is $2"); } } elsif ($line =~ /^\s*RAM:\s*(\d+)\s*Bytes$/) { $NODE_INFO{$lnn}{'RAM'} = $1; debug("NODE_INFO->${lnn}->RAM=$1"); } } } fail("This cluster contains Generation 6 nodes that may be impacted by KB 519119") if ($kb519119); info("Refer to KB 335092 (https://www.dell.com/support/kbdoc/335092) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkHardwareStatus $CHECKS{'checkMemory'}{'Description'} = "Checks each DIMM to meet criteria outlined in KB89614 and if the expected (per product info line) matches closely the reported RAM."; sub checkMemory { return if wasCalled(); checkHardwareStatus(); gatherProductInfo(); printCheck("Memory"); checkExpectedMemory(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array '${SUDO} isi_dmilog |grep DIMM'; ]. qq[sleep 2; ]. qq[2>&1 ${SUDO} isi_for_array '${SUDO} isi_hwmon -l|grep DIMM'; ]. qq[sleep 2; ]. qq[2>&1 ${SUDO} isi_for_array '${SUDO} isi_hwmon -e|grep DIMM'; ]); } else { $gather = gatherCluster("2>&1 grep -H DIMM */isi_dmilog */hwmon_ecc"); } # Analyze information for issues. my %dimm_report = (); foreach my $lnn (@LNNS) { verboseHeader("$NODE_INFO{$lnn}{'Name'} - checkMemory"); next unless $gather->{$lnn}; # Check for DIMM errors my %dimm_errors = (); foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /^\s*(DIMM\d+-RANK\d+):\s+(\d+)$/) { $dimm_report{$lnn}{$1}{$RPS{'Runtime'}}{'Count'} = $2; $dimm_report{$lnn}{$1}{$RPS{'Runtime'}}{'Type'} = 'Correctable'; verbose($line); next; } next unless $line =~ /^\s+(\d+)\/(\d+)\/(\d+)\s+(\d+):(\d+):(\d+)\s+\S+\s+(\S.*?):\s+(\d+)\s+times? on\s+(\S+)$/; debug("checkMemory: Node $lnn: $line"); my $month = $1; my $day = $2; my $year = $3; my $hour = $4; my $min = $5; my $sec = $6; my $type = $7; my $count = $8; my $dimm = $9; # Sometimes isi_dmilog has reports on 01/00/00 0:00:00.. ignoring these because there is no time stamp next if ($day == 0); # Fix a bug where dmi log rolls over the date by max int size * 2 # Also assign epoch time for the error to $time my $time = 0; if ($year > 100) { eval { $time = timelocal($sec, $min, $hour, $day, $month - 1, $year + 1832); }; warning($@) if $@; next if !$time; $time += 1006623105; ($sec, $min, $hour, $day, $month, $year) = localtime($time); $year += 1900; $month += 1; debug("checkMemory: Time conversion performed, actual date & time: $month/$day/$year $hour:$min:$sec"); } else { $year += 2000; eval { $time = timelocal($sec, $min, $hour, $day, $month - 1, $year); }; warning($@) if $@; next if !$time; } verbose("DIMM: $dimm, Count: $count, Type: $type, Time: $month/$day/$year $hour:$min:$sec"); # Skip if more than 1 year old next if ($RPS{'Runtime'} - $time >= 30758400); next if ($RPS{'Runtime'} - $time >= 7776000); $dimm_report{$lnn}{$dimm}{$time}{'Type'} = $type; $dimm_report{$lnn}{$dimm}{$time}{'Count'} = $count; my $date_str = sprintf("%04d-%02d-%02d", $year, $month, $day); $dimm_errors{$dimm}{$type}{$date_str} += $count; } foreach my $dimm (sort keys %dimm_errors) { next unless $dimm_errors{$dimm}; foreach my $type (sort keys %{$dimm_errors{$dimm}}) { next unless $dimm_errors{$dimm}{$type}; my $str = "Node $lnn DIMM $dimm is reporting $type with the following dates(count):"; foreach my $date (sort keys %{$dimm_errors{$dimm}{$type}}) { $str .= " ${date}($dimm_errors{$dimm}{$type}{$date})" } warning($str); } } } foreach my $lnn (sort {$a <=> $b} keys %dimm_report) { # Different generations of hardware have different allowances for DIMM errors in a 24 hour period my $allowed = 500; if ($NODE_INFO{$lnn}{'Generation'}) { if ($NODE_INFO{$lnn}{'Generation'} eq "Gen4") { $allowed = 15; } elsif ($NODE_INFO{$lnn}{'Generation'} eq "Gen5") { $allowed = 500; } elsif ($NODE_INFO{$lnn}{'Generation'} eq "Gen6") { $allowed = 5000; } } my @dimms_with_issues = (); foreach my $dimm (keys %{$dimm_report{$lnn}}) { my @under_24_hours = (); my $fatal = 0; my $over_allowed_in_24_hours = 0; foreach my $time (sort keys %{$dimm_report{$lnn}{$dimm}}) { # Add new time push(@under_24_hours, $time); # Shift off old times shift(@under_24_hours) while ($time - $under_24_hours[0] > 86400); # Fatal errors!? $fatal = 1 if not $dimm_report{$lnn}{$dimm}{$time}{'Type'} =~ /correctable|non-fatal/i; # Maximum of 15 errors on a DIMM allowed within a 24-hour period my $error_count = 0; $error_count += $dimm_report{$lnn}{$dimm}{$_}{'Count'} foreach (@under_24_hours); $over_allowed_in_24_hours = 1 if $error_count > $allowed; } push(@dimms_with_issues, $dimm) if $fatal or $over_allowed_in_24_hours; } fail("Node $lnn has DIMMs that met criteria for replacement: ".join(",",@dimms_with_issues)) if @dimms_with_issues; } info("Refer to KB 335088 (https://www.dell.com/support/kbdoc/335088) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkMemory ### # checkMirrors: Boot Drives - Cannot perform upgrade with degraded boot drive $CHECKS{'checkMirrors'}{'Description'} = 'Checks the boot mirror health'; sub checkMirrors { if (wasCalled()) { return } if (isVirtual()) { return unsupported('Supported on hardware platforms'); } printCheck('Mirror Status'); # Gather information from cluster or from logset my $gather = (); my $bootdisk_status = (); if ($RPS{'Live'}) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array gmirror status"); $bootdisk_status = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi_bootdisk_status]); } else { $gather = gatherCluster("2>&1 grep -H . */gmirror_status"); $bootdisk_status = gatherCluster(qq[2>&1 grep -H . */isi_bootdisk_status]); } # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); next if $NODE_INFO{$lnn}{'Accelerator'}; next if $NODE_INFO{$lnn}{'Generation'} and $NODE_INFO{$lnn}{'Generation'} eq "IQ"; if (!$gather->{$lnn} or !$bootdisk_status->{$lnn}) { missing($lnn); next; } # Check isi_bootdisk_status for missing partitions my @missing_partitions = (); foreach my $line (@{$bootdisk_status->{$lnn}}) { my @results = split(/\s+/, $line); push(@missing_partitions, $results[0]) unless $#results; } fail("Node $lnn is missing the following partitions in isi_bootdisk_status output: ".join(", ", @missing_partitions)) if @missing_partitions; my $name =""; my $status = ""; my %mirrors = (); foreach my $line (@{$gather->{$lnn}}) { verbose($line); next if ($line =~ /\s*Name\s*Status\s*Components/); if ($line =~ /(mirror\/\S+)\s+(\S+)\s+(\S+)\s*/) { $name = $1; $mirrors{$name}{'status'} = $2; push(@{$mirrors{$name}{'components'}}, $3); } elsif ($line =~ /^\s+(\S+)/) { push(@{$mirrors{$name}{'components'}}, $1) if $mirrors{$name}; } } if (%mirrors) { $NODE_INFO{$lnn}{'Bootflash'} = 1; debug("NODE_INFO->$lnn->Bootflash=1"); } my @mirrors_missing_components = (); my @mirrors_degraded = (); foreach my $name (keys %mirrors) { my $components = scalar(@{$mirrors{$name}{'components'}}); debug("checkMirrors: Node $lnn: Mirror: $name has a status: $mirrors{$name}{'status'} and $components components"); # Degraded mirrors if ($mirrors{$name}{'status'} eq "DEGRADED") { push(@mirrors_degraded, $name); } # Only one component if($components < 2 and $name !~ /crash|mfg|keystore|dump/) { push(@mirrors_missing_components, $name); } } fail("Node $lnn is reporting degraded for the following mirrors: ".join(",", @mirrors_degraded)) if (@mirrors_degraded); fail("Node $lnn is reporting only one component for the following mirrors: ".join(",", @mirrors_missing_components)) if (@mirrors_missing_components); } info("Refer to KB 335091 (https://www.dell.com/support/kbdoc/335091) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkMirrors $CHECKS{'checkNodeCompatibility'}{'Description'} = 'Checks node compatibility'. ' for OneFS upgrades by comparing it against known supported versions'; sub checkNodeCompatibility { if (wasCalled()) { return } if (!isDestinationOneFS()) { return unsupported('Supported when using the target_version parameter'); } gatherProductInfo(); if ($CLUSTER_INFO{'IQ'}) { checkPartitions() } # Setup printCheck('Node Compatibility'); my $SPF_ISERIES = 'The following i-series nodes are not compatible with'. ' OneFS 7 or later: %s'; my $SPF_ROOT = 'The following nodes do not have a sufficient root'. ' partition to support OneFS %s: %s'; my $SPF_MAX = 'The following nodes of type %s are compatible up to'. ' OneFS %s: %s'; my $MSG_NOT_COMPAT = 'Due to nodes not being compatible with the'. ' destination OneFS version, the destination OneFS version was removed'. ' and no upgrade plan will be generated.'; my $MSG_DOC = 'For more details review the Isilon Supportability and'. ' Compatibility Guide [https://support.emc.com/docu44518]'; my $SPF = '%5s %-15s %-10s %s'; my %CompatibilityMatrix = ( # IQ Node types 'IQ 1920e' => '5.5', 'IQ 3000e' => '5.5', 'IQ 4800e' => '5.5', 'IQ 6000e' => '5.5', 'IQ 200' => '5.5', 'IQ 1920i' => '6.5', 'IQ 3000i' => '6.5', 'IQ 6000i' => '6.5', 'IQ 9000i' => '6.5', 'IQ 12000i' => '6.5', 'IQ 1920x' => '7.1', 'IQ 3000x' => '7.1', 'IQ 6000x' => '7.1', 'IQ 9000x' => '7.1', 'IQ 36NL' => '7.1', 'IQ 72NL' => '7.1', 'IQ 12000x' => '7.1', 'IQ 10000x-ssd' => '8.0.0.6', 'IQ 32000x-ssd' => '8.0.0.6', 'IQ 36000x' => '8.0.0.6', 'IQ 72000x' => '8.0.0.6', 'Accelerator' => '8.0.0.6', 'IQ 5000s-ssd' => '8.0.0.6', 'IQ 5400s' => '8.0.0.6', 'IQ 108NL' => '8.0', # Gen 4 'X200' => '8.1', 'S200' => '8.1', 'X400' => '8.1', 'NL400' => '8.1', # Gen 5 'A100' => 'TBD', 'HD400' => 'TBD', 'S210' => 'TBD', 'X210' => 'TBD', 'X410' => 'TBD', 'NL410' => 'TBD', # Gen 6 'F800' => 'TBD', 'F810' => 'TBD', 'H600' => 'TBD', 'H5600' => 'TBD', 'H500' => 'TBD', 'H400' => 'TBD', 'A200' => 'TBD', 'A2000' => 'TBD', # Gen 6 MLK 'A300' => 'TBD', 'A3000' => 'TBD', 'H700' => 'TBD', 'H7000' => 'TBD', 'A250' => 'TBD', 'A2050' => 'TBD', 'F850' => 'TBD', 'H550' => 'TBD', 'H5650' => 'TBD', # Gen 6.5 'F200' => 'TBD', 'F600' => 'TBD', 'F900' => 'TBD', 'P100' => 'TBD', 'B100' => 'TBD', # Virtual 'IQ VMWare' => '8.1', 'V200' => 'TBD', 'SIMULATOR' => 'TBD', # Isilon SD Edge virtual nodes 'V100' => '8.1', ); # Analyze verboseHeader('Node Compatibility'); verbose(sprintf($SPF, 'LNN', 'Node Type', 'Max OneFS', 'Product')); verbose(sprintf($SPF, '-'x5, '-'x15, '-'x10, '-'x30)); my $delete_target = 0; my @i_series = (); my @root = (); my %max = (); foreach my $lnn (@LNNS) { # Node Product details must be available if (!$NODE_INFO{$lnn}{'Product'}) { next } my $product = $NODE_INFO{$lnn}{'Product'}; # Search for the node type in the compatibility matrix my ($type) = grep {$product =~ /^$_-/i} sort keys %CompatibilityMatrix; if (!$type) { ($type) = grep {$product =~ /$_/i} sort keys %CompatibilityMatrix; } if (!$type) { # The node type was not found in the compatibility matrix verbose(sprintf($SPF, $lnn, 'Unknown', 'Unknown', $product)); next; } # The node type was found in the compatibility matrix # Store the maximum OneFS version my $max = $CompatibilityMatrix{$type}; verbose(sprintf($SPF, $lnn, $type, $max, $product)); # Skip to the next node if there is no max if ($max eq 'TBD') { next } # Perform compatibility checks if ($NODE_INFO{$lnn}{'Series'} and $NODE_INFO{$lnn}{'Series'} =~ /i_series/ and isDestinationOneFS('>=7') and compareVersions($max, '7') >= 0) { # i Series nodes are not supported in OneFS 7 or later push(@i_series, $lnn); $delete_target = 1; } elsif (isDestinationOneFS(">$max")) { # The destination OneFS is greater than the maximum supported # version for that node type push(@{$max{$type}}, $lnn); $delete_target = 1; } elsif ($RPS{'Root less than 500MB'}{$lnn} and (isDestinationOneFS('8.0.0.7') or isDestinationOneFS('>=8.1'))) { # The node has less than 500MB allocated to / push(@root, $lnn); $delete_target = 1; } } if ($delete_target) { delete $RPS{'Target'}; delete $RPS{'UpgradePlan'}; fail($MSG_NOT_COMPAT); } # Flag if (@i_series) { fail(sprintf($SPF_ISERIES, compressRange(@i_series))); } if (@root) { fail(sprintf($SPF_ROOT, $RPS{'Target'}, compressRange(@root))); } if (%max) { foreach my $type (sort keys %max) { fail(sprintf($SPF_MAX, $type, $CompatibilityMatrix{$type}, compressRange(@{$max{$type}}))); } } if (isResults(&WARN)) { info($MSG_DOC) } # Output printResults(); } ### # gatherProductInfo: Gathers basic product info from isi_hw_status -i sub gatherProductInfo { if (wasCalled()){ return } my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array isi_hw_status -i]; my $CMD_LOGS = q[2>&1 grep -H . */isi_hw_status]; my $RE_VIRTUAL = qr/^V|VMWare|SIMULATOR/; my $RE_IQ = qr/^IQ/; my $RE_GEN4 = qr/^(?:S200|X400|X200|NL400)/; my $RE_A100 = qr/^A100/; my $RE_GEN5 = qr/^(?:S210|X410|X210|NL410|HD400)/; my $RE_GEN6 = qr/^(?:A2000?|H400|H500|H600|F800|F810|H5600|A250|A2050|F850|H550|H5650)/; my $RE_GEN65 = qr/^(?:A3000?|H7000?|F200|F600|F900)/; my $RE_SD = qr/^V100/; my $RE_ACCEL = qr/^(A100|P100|B100)|Accelerator/; # Gather information my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Analyze information foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /^\s*Product:\s+(\S.*)$/) { $NODE_INFO{$lnn}{'Product'} = $1; debug("NODE_INFO->${lnn}->Product=$1"); if ($NODE_INFO{$lnn}{'Product'} =~ $RE_VIRTUAL) { debug("NODE_INFO->${lnn}->Generation=Virtual"); $NODE_INFO{$lnn}{'Generation'} = 'Virtual'; push(@{$CLUSTER_INFO{'Virtual'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ $RE_IQ) { debug("NODE_INFO->${lnn}->Generation=IQ"); $NODE_INFO{$lnn}{'Generation'} = 'IQ'; push(@{$CLUSTER_INFO{'IQ'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ $RE_GEN4) { debug("NODE_INFO->${lnn}->Generation=Gen4"); $NODE_INFO{$lnn}{'Generation'} = 'Gen4'; push(@{$CLUSTER_INFO{'Gen4'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ $RE_A100) { push(@{$CLUSTER_INFO{'A100'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ $RE_GEN5) { debug("NODE_INFO->${lnn}->Generation=Gen5"); $NODE_INFO{$lnn}{'Generation'} = 'Gen5'; push(@{$CLUSTER_INFO{'Gen5'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ $RE_GEN6) { debug("NODE_INFO->${lnn}->Generation=Gen6"); $NODE_INFO{$lnn}{'Generation'} = 'Gen6'; push(@{$CLUSTER_INFO{'Gen6'}}, $lnn); } elsif ($NODE_INFO{$lnn}{'Product'} =~ $RE_GEN65) { debug("NODE_INFO->${lnn}->Generation=Gen6.5"); $NODE_INFO{$lnn}{'Generation'} = 'Gen6.5'; push(@{$CLUSTER_INFO{'Gen6.5'}}, $lnn); } else { debug("NODE_INFO->${lnn}->Generation=Unknown"); $NODE_INFO{$lnn}{'Generation'} = 'Unknown'; } if ($NODE_INFO{$lnn}{'Product'} =~ $RE_ACCEL) { debug("NODE_INFO->${lnn}->Accelerator=1"); $NODE_INFO{$lnn}{'Accelerator'} = 1; push(@{$CLUSTER_INFO{'Accelerator'}}, $lnn); } if ($NODE_INFO{$lnn}{'Product'} =~ $RE_SD) { debug("NODE_INFO->${lnn}->Generation=IsilonSD"); $NODE_INFO{$lnn}{'Generation'} = 'IsilonSD'; push(@{$CLUSTER_INFO{'IsilonSD'}}, $lnn); } } elsif ($line =~ /^\s*Series:\s+(\S.*)$/) { $NODE_INFO{$lnn}{'Series'} = $1; debug("NODE_INFO->${lnn}->Series=$1"); } elsif ($line =~ /^\s*SerNo:\s+(\S.*)$/) { $NODE_INFO{$lnn}{'Serial'} = $1; debug("NODE_INFO->${lnn}->Serial=$1"); } elsif ($line =~ /^\s*FamCode:\s+(\S.*)$/) { if ($1 ne 'V') { next } debug("NODE_INFO->${lnn}->Generation=Virtual"); $NODE_INFO{$lnn}{'Generation'} = 'Virtual'; push(@{$CLUSTER_INFO{'Virtual'}}, $lnn); } } } } # End gatherProductInfo ### # checkBMC: checks for BMC related issues $CHECKS{'checkBMCandCMC'}{'Description'} = 'Checks for BMC/CMC related issues'; sub checkBMCandCMC { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen5'}) { return unsupported('Only supported on S210, X210, X410, HD400, and'. ' NL410 nodes and this cluster has none'); } checkHardwareStatus(); printCheck('BMC/CMC Hardware Monitoring'); gatherBMCInfo(); gatherCMCInfo(); gatherNodeFirmware(); my $MSG_UNRESPONSIVE = 'Be advised, nodes with unresponsive BMC/CMC or not'. ' reporting a BMC/CMC firmware version are assumed to be out of date'; my $MSG_LOG_BMC = 'Be advised, log gathers in OneFS 7.1.1 do not include'. ' BMC firmware versions, BMC firmware versions are assumed to be out'. ' of date'; my $SPF_PRI_BMC = 'BMC firmware updates are strongly recommended because'. ' %s nodes are affected by F010517EE'; my $MSG_RBT_BMC = 'Nodes with out of date BMC firmware have an increased'. ' risk of encountering hardware monitoring issues and may encounter'. ' issues when rebooted.'; my $SPF_FCO_BMC = '%s nodes are running BMC firmware versions affected by'. ' FCO F010517EE: %s'; my $MSG_UPG_BMC = 'BMC firmware updates cannot be performed remotely by'. ' RPS on OneFS versions prior to 8.0.0.2.'; my $SPF_FCO_CMC = '%s nodes are running CMC firmware versions affected by'. ' FCO F010517EE: %s'; my $SPF_NONE = 'The %s device is reporting None on the following nodes: %s'; my $SPF_HWMON = 'Hardware monitoring issues detected on nodes: %s'; my $MSG_KB = 'For more details see KB 466373'. ' [https://www.dell.com/support/kbdoc/466373 - Isilon S210, X210,'. ' X410, NL410 or HD400 shows event: "Node\'s Baseboard Management'. ' Controller (BMC) and/or Chassis Management Controller (CMC) are'. ' unresponsive." - Event ID: 910100006, 999910002]'; # Look for BMC/CMC issues my @node_list = (); my %none = (); foreach my $lnn (@{$CLUSTER_INFO{'Gen5'}}) { if (!$NODE_INFO{$lnn}{'Firmware'}) { next } my %fw = %{$NODE_INFO{$lnn}{'Firmware'}}; # If the CMC firmware is not up to 02.05 or unresponsive if (!$fw{'CMC'} or compareVersions($fw{'CMC'}, '02.05') < 0) { push(@{$RPS{'CMC Firmware'}{'Nodes'}}, $lnn); } # If the BMC firmware is not up to 01.25 or unresponsive if (!$fw{'BMC'} or compareVersions($fw{'BMC'}, '01.25') < 0) { push(@{$RPS{'BMC Firmware'}{'Nodes'}}, $lnn); } # Check each node for hardware monitoring issues my $hardware_monitoring_issue = 0; # BMC or CMC is not found/unresponsive if ($RPS{'Live'} or isCurrentOneFS('>7.2')) { if (!$fw{'BMC'}) { debug("checkBMCandCMC: !!! Node $lnn BMC not responsive !!!"); $hardware_monitoring_issue = 1; } if (!$fw{'CMC'}) { debug("checkBMCandCMC: !!! Node $lnn CMC not responsive !!!"); $hardware_monitoring_issue = 1; } } # Both power supplies don't report FW versions if (!$fw{'PS1'} and !$fw{'PS2'}) { debug("checkBMCandCMC: !!! Node $lnn both PS not reporting !!!"); $hardware_monitoring_issue = 1; } # Any ^(BMC/CAR/CFFPS/CMC/CMD) component reports None my @results = grep {$_ =~ /^(?:BMC|CAR|CMC|CMD)_/} keys %fw; foreach my $device (@results) { if ($fw{$device} eq 'None') { $hardware_monitoring_issue = 1; push(@{$none{$device}}, $lnn); debug("checkBMCandCMC: !!! Node $lnn Device $device is". " reporting None"); } } # Add to list of nodes with hardware monitoring issues if ($hardware_monitoring_issue) { debug("checkBMCandCMC: Node $lnn found hardware monitoring issue"); push(@node_list, $lnn); } } if ($RPS{'BMC Firmware'}{'Nodes'} and $RPS{'CMC Firmware'}{'Nodes'}) { # BMC firmware updates must be performed before node firmware updates $RPS{'BMC First'} = 1; } if ($RPS{'BMC Firmware'}{'Nodes'} or $RPS{'CMC Firmware'}{'Nodes'}) { # BMC/CMC is assumed out of date if not responsive/gathered if (@node_list) { info($MSG_UNRESPONSIVE); } # BMC is assumed to be out of date for log gathers in OneFS 7.1.1 if (!$RPS{'Live'} and isCurrentOneFS(qr/^7\.1\.1\./)) { info($MSG_LOG_BMC); } } # BMC firmware out of date on some nodes if ($RPS{'BMC Firmware'}{'Nodes'}) { # Flag BMC firmware FCO details $RPS{'FCO'}{'F010517EEB'}{'Severity'} = 'INFO'; $RPS{'FCO'}{'F010517EEB'}{'Message'} = sprintf($SPF_FCO_BMC, scalar(@{$RPS{'BMC Firmware'}{'Nodes'}}), compressRange(@{$RPS{'BMC Firmware'}{'Nodes'}})); critical($RPS{'FCO'}{'F010517EEB'}{'Message'}); critical($MSG_RBT_BMC); # BMC Firmware Activity $RPS{'BMC Firmware'}{'Target'} = 'BMC Firmware Package 1.0'; push(@{$RPS{'BMC Firmware'}{'Files'}}, 'Isilon BMC Firmware Package 1.0'); push(@{$RPS{'BMC Firmware'}{'Priority'}}, sprintf($SPF_PRI_BMC, scalar(@{$RPS{'BMC Firmware'}{'Nodes'}}))); # Set the position of where the BMC firmware updates will be completed if (isCurrentOneFS('>=8.0.0.2')) { $RPS{'BMC Firmware'}{'Position'} = 1; } elsif (isDestinationOneFS('>=8.0.0.2')) { $RPS{'BMC Firmware'}{'Position'} = 2; } else { warning($MSG_UPG_BMC); } } # Flag CMC firmware that is lower than required for FCO F010517EEC if ($RPS{'CMC Firmware'}{'Nodes'}) { $RPS{'FCO'}{'F010517EEC'}{'Severity'} = 'INFO'; $RPS{'FCO'}{'F010517EEC'}{'Message'} = sprintf($SPF_FCO_CMC, scalar(@{$RPS{'CMC Firmware'}{'Nodes'}}), compressRange(@{$RPS{'CMC Firmware'}{'Nodes'}})); info($RPS{'FCO'}{'F010517EEC'}{'Message'}); } # Flag CMC/BMC components/devices reporting None foreach my $device (keys %none) { fail(sprintf($SPF_NONE, $device, compressRange(@{$none{$device}}))); } # Flag hardware monitoring issues if (@node_list) { fail(sprintf($SPF_HWMON, compressRange(@node_list))) } # Output if (isResults(&WARN)) { info($MSG_KB) } printResults(); } # End checkBMCandCMC $CHECKS{'checkBXENodes'}{'Description'} = "Checks for existence of nodes with BXE interfaces, also checks for KB493493 & KB511208 known issues."; sub checkBXENodes { if (wasCalled()) { return } # Pre-requisite version checks if (isCurrentOneFS('<7.1.1')) { debug("checkBXENodes: Skipped.."); return; } return unsupported("Unsupported on virtual platforms") if isVirtual(); # Print showing check running printCheck("BXE Nodes"); # Gather required information gatherNetworkInterfaces(); # Flag nodes with BXE interfaces or that no nodes have BXE interfaces my $num_bxe_nodes = 0; if ($CLUSTER_INFO{'BXE Nodes'}) { $num_bxe_nodes = scalar(@{$CLUSTER_INFO{'BXE Nodes'}}); details("$num_bxe_nodes nodes have BXE interfaces: ".compressRange(@{$CLUSTER_INFO{'BXE Nodes'}})); } # Flag issue in KB 493493 checkKB493493(); # Flag issue in KB 511208 checkKB511208(); # Output results printResults(); } # End checkBXENodes $CHECKS{'checkKB462202'}{'Description'} = 'Checks BootOrder in bios_settings.ini on Generation 5 nodes to determine if at risk for KB 462202'; sub checkKB462202 { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen5'}) { return unsupported('Supported for Generation 5 nodes only') } printCheck('KB 462202'); my $HDD = 'Hard Drive'; my $EFI = 'Internal EFI Shell'; my $MSG_KB462202 = 'For more details see KB 462202 [https://www.dell.com/support/kbdoc/462202 - Gen5 (S210, X210, X410, HD400) node shows EFI shell prompt instead of booting]'; my $SPF_NODES_AT_RISK = 'The boot order has "Internal EFI Shell" prior to "Hard Drive" on the following nodes: %s'; my $CMD_NODE = qq[grep -E "$HDD|$EFI" /var/log/bios_settings.ini]; my $CMD_LOGS = qq[grep -H -E "$HDD|$EFI" */varlog.tar/log/bios_settings.ini]; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array '$CMD_NODE']); } else { $gather = gatherCluster(qq[2>&1 $CMD_LOGS]); } # Analyze information for issues. my %boot_order = (); my @nodes_at_risk = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/($HDD|$EFI)=(\d+)$/) { $boot_order{$lnn}{$1} = $2; } } if (!$boot_order{$lnn}{$HDD} or !$boot_order{$lnn}{$EFI}) { next; } if ($boot_order{$lnn}{$HDD} > $boot_order{$lnn}{$EFI}) { push(@nodes_at_risk, $lnn); } } # Flag any issues identified if (@nodes_at_risk) { fail(sprintf($SPF_NODES_AT_RISK, compressRange(@nodes_at_risk))); info($MSG_KB462202); } # Output Check result printResults(); } # End checkKB462202 sub checkKB493493 { if (wasCalled()) { return } if (!isCurrentOneFS(qr/^8\.0\.0\.[01234]$/)) { return } # Gather NIC driver if available and patches information gatherPatches(); gatherBXEDriver(); gatherNetworkInterfaces(); my $SPF_INTERFACES = '%s interface affected by KB493493 on nodes: %s'; my $SPF_PATCH_REBOOT = 'Patch 188239 is installed, but the following nodes were detected as running old NIC drivers which need to be rebooted with lagg interfaces first being taken down on each node prior to reboot: %s'; my $SPF_PATCH_CONFLICT = 'Prior to installing patch-188239, the conflicting patch, %s, must be uninstalled. A reboot of nodes is not neccesary after the uninstall of the conflicting patch, but is neccesary after the install of patch-188239.'; my $MSG_REASON = 'Installation of patch-188239 is required to avoid node panics caused by issues outlined in KB493493.'; my $MSG_LAGG = 'Prior to rebooting nodes for the installation of patch-188239, the lagg interfaces on the node to be rebooted must be taken down to avoid node panics during the reboots. If rebooting all nodes simultaneously, avoid taking down all lagg interfaces as this may leave the cluster in an inaccessible state.'; # Based on the code level, try to make assumptions on the BXE NIC driver my $safe_driver_codelevel = 0; if (isCurrentOneFS(qr/^8\.0\.0\.[012]$/)) { $safe_driver_codelevel = 1; } if (hasPatch('188239')) { $safe_driver_codelevel = 1; } if (hasPatch('182721')) { $safe_driver_codelevel = 0; } # Check for conflicting patch installs my ($patch_conflict) = hasPatch('182721|175450|171054'); # Check to see if any interfaces are impacted my %lagg_interfaces = (); foreach my $lnn (@LNNS) { # Not running an impacted NIC driver version if ($NODE_INFO{$lnn}{'dev.bxe.0.version'} and $NODE_INFO{$lnn}{'dev.bxe.0.version'} !~ /^1\.78\.89\.ISILON\.0[234]$/) { next; } # Unable to gather BXE nic information due to running on a log gather. # Make assumptions from current code level. if (!$NODE_INFO{$lnn}{'dev.bxe.0.version'} and $safe_driver_codelevel) { next; } my @affected_interfaces = (); foreach my $interface (keys %{$NODE_INFO{$lnn}{'Network'}}) { if ($interface !~ /lagg/i) { next } if (!$NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'}) { next } if ($NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} !~ /lacp/i) { next } if (!$NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'}) { next } my @results = grep {$_ =~ /^bxe/} @{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'}}; if (@results) { push(@affected_interfaces, $interface) } } foreach (@affected_interfaces) { push(@{$lagg_interfaces{$_}}, $lnn); } if (@affected_interfaces) { push(@{$RPS{'KB493493'}}, $lnn); } } foreach (keys %lagg_interfaces) { warning(sprintf($SPF_INTERFACES, $_, compressRange(@{$lagg_interfaces{$_}}))); } # If identified as impacted by KB493493, patch 188239 should be performed prior to any node reboots if ($RPS{'KB493493'}) { if (hasPatch('188239')) { @{$RPS{'Proactive Reboot'}{'Nodes'}} = @{$RPS{'KB493493'}}; addCustomerInformation('Upgrade Plan', warning(sprintf($SPF_PATCH_REBOOT, compressRange(@{$RPS{'KB493493'}})))); } else { # Provide a reason why we are doing the patch/reboot addCustomerInformation('Upgrade Plan', $MSG_REASON); # This will make sure it gets into the upgrade plan $RPS{'Pre Upgrade Patches'}{'patch-188239'}{'Impacts'} = ''; @{$RPS{'Pre Upgrade Patches'}{'patch-188239'}{'Reboot'}} = @{$CLUSTER_INFO{'BXE Nodes'}}; if ($patch_conflict) { push(@{$RPS{'Pre Upgrade Patches'}{'patch-188239'}{'Additional Comments'}}, warning(sprintf($SPF_PATCH_CONFLICT, $patch_conflict))); } push(@{$RPS{'Pre Upgrade Patches'}{'patch-188239'}{'Additional Comments'}}, warning($MSG_LAGG)); } } } # End checkKB493493 sub checkKB511208 { return if wasCalled(); return if (isCurrentOneFS(qr/^8\.0\.0/) and isCurrentOneFS('>=8.0.0.7')) or (isCurrentOneFS(qr/^8\.1\.0/) and isCurrentOneFS('>=8.1.0.4')) or isCurrentOneFS('>=8.1.1.1'); gatherNetworkInterfaces(); # Analyze information for issues. my @impacted = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Network'}; my @lagg_interfaces = grep {$_ =~ /^lagg/} keys %{$NODE_INFO{$lnn}{'Network'}}; foreach my $lagg_interface (@lagg_interfaces) { next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'} =~ /lacp/; foreach my $laggport (@{$NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}}) { my $interface = ""; my $flags = ""; $interface = $1 if $laggport =~ /^(\S+)/; $flags = $1 if $laggport =~ /flags=(\S+)/; next unless $interface and $interface =~ /bxe/; next unless $flags; next unless $NODE_INFO{$lnn}{'Network'}{$interface}{'status'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'status'} eq "active"; next unless $flags =~ /0<>/; # next if $flags =~ /ACTIVE/ and $flags =~ /COLLECTING/ and $flags =~ /DISTRIBUTING/; push(@impacted, "Node $lnn, aggregate interface $lagg_interface is reporting 'flags=0<>' for the interface $interface"); } } } # Skip unless identified as impacted return unless @impacted; if (isDestinationOneFS('8.0.0.7') or isDestinationOneFS('8.1.0.4') or isDestinationOneFS('>=8.1.1.1')) { warning(@impacted, "The above listed interfaces are currently impacted by KB511208. This issue is addressed in OneFS 8.0.0.7, OneFS 8.1.0.4, OneFS 8.1.1.1 and later codes."); } else { fail(@impacted, "The above listed interfaces are currently impacted by KB511208. This issue is addressed in OneFS 8.0.0.7, OneFS 8.1.0.4, OneFS 8.1.1.1 and later codes."); } } # End checkKB511208 # gatherBMCInfo: Gather BMC firmware information sub gatherBMCInfo { if (wasCalled()) { return } if (!$CLUSTER_INFO{'Gen5'}) { return } my $nodes = compressRange(@{$CLUSTER_INFO{'Gen5'}}); if (!$nodes) { return } my $CMD_LIVE = qq[ 2>&1 ${SUDO} isi_for_array -n ${nodes} ]. qq[ '${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -d -V -a bmc' ]. q[ | grep firmware ]; my $CMD_LOGS = q[ 2>&1 grep -H firmware */bmc_info ]; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Store foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/IPMI firmware version = (.*)$/) { $NODE_INFO{$lnn}{'Firmware'}{'BMC'} = $1; debug("NODE_INFO->${lnn}->Firmware->BMC=$1"); } } } } ### # gatherBXEDriver: Gather BXE NIC driver version sub gatherBXEDriver { if (wasCalled()) { return } if (!$CLUSTER_INFO{'Gen5'}) { return } my $nodes = compressRange(@{$CLUSTER_INFO{'Gen5'}}); if (!$nodes) { return } my $CMD_LIVE = qq[ 2>&1 ${SUDO} isi_for_array -n $nodes ]. q[ 'sysctl dev.bxe.0.version' ]; my $CMD_LOGS = q[ 2>&1 zgrep -H 'bxe0.*BXE.*ISILON' ]. q[ */varlog.tar/log/messages* ]; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Store foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach my $line (@{$gather->{$lnn}}) { my $found = ''; if ($line =~ /^dev.bxe.0.version:\s*(\S+)$/) { $found = $1; } elsif ($line =~ /BXE v:(.*?)>/) { $found = $1; } if (!$found) { next } if ($NODE_INFO{$lnn}{'dev.bxe.0.version'}) { my $existing = $NODE_INFO{$lnn}{'dev.bxe.0.version'}; if (compareVersions($existing, $found) >= 0) { next } } $NODE_INFO{$lnn}{'dev.bxe.0.version'} = $found; debug("NODE_INFO->${lnn}->dev.bxe.0.version=$found"); } } } # End gatherBXEDriver # gatherCMCInfo: Gather CMC firmware information sub gatherCMCInfo { if (wasCalled()) { return } if (!$CLUSTER_INFO{'Gen5'}) { return } my $nodes = compressRange(@{$CLUSTER_INFO{'Gen5'}}); if (!$nodes) { return } my $CMD_LIVE = qq[ 2>&1 ${SUDO} isi_for_array -n $nodes ]. qq[ '${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -d -V -a cmc' ]. qq[ | grep firmware ]; my $CMD_LOGS = q[2>&1 grep -H firmware */cmc_info]; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Store foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/IPMI firmware version = (.*)$/) { $NODE_INFO{$lnn}{'Firmware'}{'CMC'} = $1; debug("NODE_INFO->${lnn}->Firmware->CMC=$1"); } } } } $CHECKS{'checkKB519119'}{'Description'} = "Checks if nodes may be impacted by KB 519119"; sub checkKB519119 { return if wasCalled(); return unless $RPS{'Live'}; gatherProductInfo(); return unless $CLUSTER_INFO{'Gen6'}; printCheck("KB519119"); # Gather information from Gen6 nodes my $gather = gatherCluster (qq[2>&1 ${SUDO} isi_for_array -n ].compressRange(@{$CLUSTER_INFO{'Gen6'}}). qq[ 'echo; ]. qq[ echo PS0-A; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b005020000ff10f50050; ]. qq[ echo PS0-B; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b005020000ff10f50060; ]. qq[ echo PS0-C; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b005020000ff10f50070; ]. qq[ echo PS1-A; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b006020000ff10f50050; ]. qq[ echo PS1-B; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b006020000ff10f50060; ]. qq[ echo PS1-C; 2>&1 ${SUDO} /usr/bin/isi_hwtools/isi_ipmicmc -V -a bmc -R 30b006020000ff10f50070 ]. qq[ ' ]); # Expected RAW IPMI responses my %expected = ("PS0-A" => "b0 00 10 4d 46 52 5f 52 45 53 50 2a 00 00 40 00 00 00 18", "PS0-B" => "b0 00 10 4d 46 52 5f 43 41 4c 90 2a 00 00 00 01 00 00 31", "PS0-C" => "b0 00 10 4d 46 52 5f 46 57 55 90 2b 00 00 20 d4 00 00 1b", "PS1-A" => "b0 00 10 4d 46 52 5f 52 45 53 50 2a 00 00 40 00 00 00 18", "PS1-B" => "b0 00 10 4d 46 52 5f 43 41 4c 90 2a 00 00 00 01 00 00 31", "PS1-C" => "b0 00 10 4d 46 52 5f 46 57 55 90 2b 00 00 20 d4 00 00 1b"); # Analyze each node for issues my %nodes = (); foreach my $lnn (@{$CLUSTER_INFO{'Gen6'}}) { if (!$gather->{$lnn}) { missing($lnn); next; } my $cmd = ""; my $response = ""; my $valid_responses = 0; foreach my $line (@{$gather->{$lnn}}) { # Gather which PS is being checked $cmd = $1 if $line =~ /(PS[01]-[ABC])/; next unless $cmd and $expected{$cmd}; # Gather raw ipmi response and compare to expected response if ($line =~ /^\s*((?:[0-9a-f]{2}\s)*[0-9a-f]{2})$/) { $response .= ($response?" $1":$1); } else { $response = ""; } next unless $response and $expected{$cmd} eq $response; # Increment the number of valid responses found for this node $cmd = ""; $response = ""; $valid_responses++; } # If there was not 6 responses, the EEPROM may be corrupted debug("checkKB519119: Number of valid responses found on node $lnn: $valid_responses"); if ($valid_responses ne 6) { $nodes{$lnn} = 1; } } # Flag if issues detected if (%nodes) { fail("The following nodes may be impacted by KB 519119: ".compressRange(keys %nodes)); fail("Run the script as outlined in KB 519119 to verify."); fail("Nodes idenfitied as affected by KB 519119 should have the impacted power supply replaced prior to performing any node firmware updates."); } # Output Check result printResults(); } # End checkKB519119 $CHECKS{'checkKB540513'}{'Description'} = 'Checks for criteria of KB 540513'; sub checkKB540513 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2') or isCurrentOneFS('<8.1')) { return unsupported('Supported in OneFS 8.1') } if (!isDestinationOneFS('<8.2')) { return unsupported('Supported when target_version is OneFS 8.1'); } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return unsupported('Supported on Generation 6 hardware'); } # Setup printCheck('KB 540513'); my $SPF_NODES = 'The following nodes have serial numbers which will cause'. ' OneFS upgrades to OneFS 8.1 releases to fail during the upgrade'. ' commit: %s'; my $MSG_FIXED = 'This issue has been addressed in the February 2020'. ' pre-patched installation file for OneFS 8.1.2.0. If performing an'. ' upgrade to OneFS 8.1.2.0, ensure you utilize a pre-patched'. ' installation file dated after February 2020'; my $MSG_KB = 'For more information please review KB540513'. ' [https://www.dell.com/support/kbdoc/540513 - {ISILON} 8.1+ OneFS'. ' upgrade from 8.1.X fails with a "Invalid node serial number" error'. ' for clusters with Gen6 nodes'; # Analyze my @nodes = (); foreach my $lnn (@{$CLUSTER_INFO{'Gen6'}}) { if (!$NODE_INFO{$lnn}{'Serial'}) { missing($lnn); next; } if ($NODE_INFO{$lnn}{'Serial'} !~ '^\S{5}[0-9]{9}$') { push(@nodes, $lnn); } } # Flag if (@nodes) { if (isDestinationOneFS('8.1.2.0')) { critical(sprintf($SPF_NODES, compressRange(@nodes))); critical($MSG_FIXED); } else { fail(sprintf($SPF_NODES, compressRange(@nodes))); } info($MSG_KB); } # Output printResults(); } # End checkKB540513 $CHECKS{'checkKB540901'}{'Description'} = 'Checks boot disk partitions for any mismatches in uuids which may lead to boot failures'; sub checkKB540901 { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return unsupported('Supported for Generation 6 nodes only'); } printCheck('KB 540901'); my $CMD_LOGS = 'grep -E "<(name|type|rawuuid)>" */kern.geom | grep -C1 bootdiskid'; my $CMD_NODE = 'sysctl kern.geom | grep -E "<(name|type|rawuuid)>" | grep -C1 bootdiskid'; my $SPF_NODES_AT_RISK = 'The following nodes have boot disk partitions with different uuids which may lead to boot failures: %s'; my $MSG_KB540901 = 'For more details see KB 540901 [https://www.dell.com/support/kbdoc/540901 - Node boot hindered by differing "bootdiskid" partition IDs]'; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array '$CMD_NODE']); } else { $gather = gatherCluster(qq[2>&1 $CMD_LOGS]); } # Analyze information for issues. my @nodes_at_risk = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } my %uuids = (); foreach (@{$gather->{$lnn}}) { if (/.*?-([^-]+) 1) { push(@nodes_at_risk, $lnn); } } # Flag any issues identified if (@nodes_at_risk) { fail(sprintf($SPF_NODES_AT_RISK, compressRange(@nodes_at_risk))); info($MSG_KB540901) } # Output check results printResults(); } # End checkKB540901 $CHECKS{'checkKB546604'}{'Description'} = 'Checks for criteria of KB 546604'; sub checkKB546604 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2')) { return unsupported('Supported in OneFS 8.1'); } if (!isDestinationOneFS('>=9')) { return unsupported('Supported in upgrades to OneFS 9 or later'); } if (isDestinationOneFS('>=9.1.0.5')) { return unsupported('Fixed in OneFS 9.1.0.5 and newer.'); } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return unsupported('Supported for Generation 6 nodes'); } gatherNetworkInterfaces(); if (!$CLUSTER_INFO{'Active IB Interfaces'}) { return unsupported('Supported for InfiniBand back end network'); } # Setup printCheck('KB 546604'); my $SPF_NODES = "During the upgrade from $CLUSTER_INFO{'OneFS'} to". " $RPS{'Target'}, the following nodes will lose front end connectivity". ' on the mlxen network interfaces until the upgrade has been'. ' committed: %s. To avoid this, it is recommended to upgrade to a'. " OneFS 8.2 release as an interim jump to get to $RPS{'Target'}.". ' This issue has also been addressed in OneFS 9.1.0.5. Consider'. ' instead upgrading to the latest OneFS 9.1.0 patch release.'; # Analyze my @nodes = (); foreach my $lnn (@{$CLUSTER_INFO{'Gen6'}}) { if (!$NODE_INFO{$lnn}{'Network'}) { next } my ($match) = grep(/mlxen[01]/, keys %{$NODE_INFO{$lnn}{'Network'}}); if ($match) { push(@nodes, $lnn) } } # Flag if (@nodes) { my $msg = sprintf($SPF_NODES, compressRange(@nodes)); addCustomerInformation('Upgrade Plan', critical($msg)); $RPS{'KB546604'} = 1; } # Output printResults(); } sub checkVaultCardCamcontrol { if (wasCalled()) { return } if (isLive() and isCompliance() and !$RPS{'JSON'}) { return; } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return } my $NODES = compressRange(@{$CLUSTER_INFO{'Gen6'}}); my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array -n $NODES '${SUDO} camcontrol devlist']. qq[| grep ada0]; my $CMD_LOGS = q[2>&1 grep -H ada0 */camcontrol]; my $SPF_FAIL = 'The vault card located at ada0 in the following Generation 6 nodes '. 'are not reporting in the output of camcontrol devlist: %s'; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Analyze information for issues. my @nodes = (); foreach my $lnn (@LNNS) { if ($NODE_INFO{$lnn}{'Generation'} ne 'Gen6') { next } if ($gather->{$lnn}) { my @results = grep /\(.*ada0.*\)$/, @{$gather->{$lnn}}; if (@results) { next } } push(@nodes, $lnn); } if (@nodes) { fail(sprintf($SPF_FAIL, compressRange(@nodes))) } } sub checkVaultCardSMARTStatus { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return } my $NODES = compressRange(@{$CLUSTER_INFO{'Gen6'}}); my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array -n $NODES '${SUDO} isi_radish -a ada0']. qq[| grep 'SMART status']; my $CMD_LOGS = q[2>&1 grep -H 'SMART status' */isi_radish_bootdisk]; my $SPF_FAIL = 'The vault card located at ada0 in the following Generation 6 nodes '. 'are reporting SMART status is threshold exceeded: %s'; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Analyze information for issues. my @nodes = (); foreach my $lnn (@LNNS) { if ($NODE_INFO{$lnn}{'Generation'} ne 'Gen6') { next } if (!$gather->{$lnn}) { next } my @results = grep(/threshold exceeded.*\(.*ada0.*\)$/, @{$gather->{$lnn}}); if (@results) { push(@nodes, $lnn) } } if (@nodes) { fail(sprintf($SPF_FAIL, compressRange(@nodes))) } } $CHECKS{'checkVaultCard'}{'Description'} = 'Checks for the presence of the M.2 vault card in Generation 6 nodes and '. 'confirms SMART status threshold has not been exceeded on that device.'; sub checkVaultCard { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return unsupported('Supported on Generation 6 hardware'); } my $MSG_KB = 'For more details see KB 545370'. ' [https://www.dell.com/support/kbdoc/545370 - [Isilon] Node Firmware'. ' updates immediately error on Generation 6 nodes with a'. ' missing/failed M2 Vault Card]'; printCheck('Vault Card'); checkVaultCardCamcontrol(); checkVaultCardSMARTStatus(); if (isResults(&INFO)) { info($MSG_KB) } printResults(); } # End checkVaultDrive ### # getGatherDate: Gather the date of the gather from package_info.xml sub getGatherDate { if (isLive()) { return 0 } if (defined $RPS{'Gather Date'}) { return $RPS{'Gather Date'} } # Setup my $CMD_LOGS = q[2>&1 cat package_info.xml]; # Gather my $gather = gatherLocal($CMD_LOGS); # Save foreach (@$gather) { if (/(.*)<\/gather_date>/) { return $RPS{'Gather Date'} = $1; } } # Bugfix: Due to a bug in some OneFS versions, the contents of # package_info.xml may not be populated. For those instances use the last # modify timestamp of package_info.xml as the gather date. if (!$RPS{'Gather Date'} and -e 'package_info.xml') { return $RPS{'Gather Date'} = (stat('package_info.xml'))[9]; } } # End getGatherDate ### # getLicense: Returns a reference to the license hash if licensed, otherwise 0 sub getLicense { my ($feature) = @_; # Return 0 if there is no license being checked if (!$feature) { debug('getLicense: Returning 0 (No arguments provided)'); return 0; } # Gather license details incase not previously gathered gatherLicense(); # Return 0 if license details were not gathered for whatever reason if (!$CLUSTER_INFO{'License'}) { debug('getLicense: Returning 0 (No license details available)'); return 0; } # Search for the licensed feature debug("getLicense: Looking for feature that contains: $feature"); ($feature) = grep {$_ =~ /$feature/i} keys %{$CLUSTER_INFO{'License'}}; # Return 0 if unable to locate licensed feature if (!$feature or !$CLUSTER_INFO{'License'}{$feature} or !$CLUSTER_INFO{'License'}{$feature}{'Status'}) { debug('getLicense: Returning 0 (No feature found)'); return 0; } # Return hash reference to license return $CLUSTER_INFO{'License'}{$feature}; } # End getLicense # getNodesOfProduct: Returns a list of nodes matching the requested product sub getNodesOfProduct { my ($re_nodes) = @_; if (!$re_nodes) { return } gatherProductInfo(); my @nodes = (); foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'Product'}) { next } if ($NODE_INFO{$lnn}{'Product'} =~ /$re_nodes/) { push(@nodes, $lnn) } } debug("getNodesOfProduct: Request: $re_nodes, Return: [ @nodes ]"); return @nodes; } ### # hasLicense: Returns 1 if licensed, otherwise 0 sub hasLicense { my ($feature) = @_; my ($license_href) = getLicense($feature); # Return 0 if there is no license found if (!$license_href) { debug('hasLicense: Returning 0 (No license found)'); return 0; } debug("hasLicense: $feature license state is: $license_href->{'Status'}"); # Return 1 if licensed if ($license_href->{'Status'} =~ /^(activated|licensed|evaluation)$/i) { debug('hasLicense: Returning 1'); return 1; } # Return 0 if not licensed debug('hasLicense: Returning 0'); return 0; } # End hasLicense ### # Initiate IOCA sub initIOCA { $RPS{'Runtime'} = time(); $RPS{'Timestamp'} = $RPS{'Runtime'}; # Parse command line arguments parseArguments(@ARGV); # Flag if this version is over 30 days old my $days_since_release = int(($RPS{'Runtime'} - $RPS{'Release Timestamp'}) / 86400); if ($days_since_release > 30) { printUrgent("This version is over $days_since_release days old, a new". ' version may be available.'); printUrgent('If FTP access is available, download the latest version'. ' with:'); printUrgent(' curl --disable-epsv -O'. ' ftp.emc.com/pub/rcm/Isilon/tools/IOCA'); } # Display current time for live cluster or logset path for a logset if (isLive()) { printCheck('Live Cluster Analysis', scalar(localtime($RPS{'Runtime'}))); } else { printCheck('Logset Path', getcwd()); if (getGatherDate()) { printCheck('Gather Date', scalar(localtime(getGatherDate()))); $RPS{'Timestamp'} = getGatherDate(); # Warn if logset gather date is over 1 month old if ($RPS{'Runtime'} - getGatherDate() > 2592000) { printUrgent('Logset is over 1 month old'); } } else { printCheck('Gather Date', 'Undetermined'); } } # Determine if the cluster is compliance mode and assign sudo variable $SUDO = (isCompliance())?'sudo ':''; # Check that script is being ran by root or compadmin if (isLive()) { my $expected_user = (isCompliance()?'compadmin':'root'); chomp(my $current_user = `whoami`); if ($current_user ne $expected_user) { printUrgent("IOCA should be ran as $expected_user"); printUrgent("Results may not be accurate"); } } # Gather and display basic node and cluster information gatherArrayXML(); if (!$CLUSTER_INFO{'Name'}) { if (isLive()) { printUrgent('Unable to gather details from /etc/ifs/array.xml'); } else { printUrgent('Unable to gather details from'. ' */etc.tar/etc/ifs/array.xml'. ' */etcifs.tar/ifs/array.xml'); printUrgent('Please ensure you are running health checks with a'. ' full gather'); } exit 1; } if ($CLUSTER_INFO{'Name'}) { printCheck('Cluster Name', $CLUSTER_INFO{'Name'}); } if ($CLUSTER_INFO{'GUID'}) { printCheck('Cluster GUID', $CLUSTER_INFO{'GUID'}); } if ($CLUSTER_INFO{'Node Count'}) { printCheck('Node Count', $CLUSTER_INFO{'Node Count'}); } if (isCompliance()) { printCheck('SmartLock Compliance Mode', 'ENABLED'); } # Gather and display current OneFS version gatherOneFSVersions(); if (isCurrentOneFS()) { printCheck('Current OneFS Version', isCurrentOneFS()); } # Check destination OneFS version checkDestinationOneFS(); # Check OneFS Version for issues # This script will not work on OneFS 5.5 and earlier if (isCurrentOneFS('<6')) { printUrgent("IOCA is not compatible with versions prior to OneFS 6\n"); exit 1; } # If a specific check is to be ran, run it and exit if ($RPS{'Run'}) { &{$_}() foreach @{$RPS{'Run'}}; if (@Logger::RESULTS and $RPS{'JSON'}) { print toJSON(\@Logger::RESULTS)."\n"; exit 0; } if (isResults()) { printCheck('Script clean-up'); printResults(); } exit 0; } } # End initIOCA ### # isCompliance: Returns 1 if the cluster is compliance mode enabled sub isCompliance { if (defined $CLUSTER_INFO{'Compliance'}) { return $CLUSTER_INFO{'Compliance'}; } $CLUSTER_INFO{'Compliance'} = 0; my $CMD_LIVE = q[2>/dev/null sysctl isi.compliance]; my $CMD_LOGS = q[2>/dev/null cat local/isi*compliance]; my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); foreach (@$gather) { if (/isi.compliance: (\d)/) { $CLUSTER_INFO{'Compliance'} = $1 } } debug("CLUSTER_INFO->Compliance=$CLUSTER_INFO{'Compliance'}"); return $CLUSTER_INFO{'Compliance'}; } # End isCompliance ### # isCurrentOneFS: Uses either Regexp or scalar to test current OneFS sub isCurrentOneFS { my ($check) = @_; gatherOneFSVersions(); if (!$CLUSTER_INFO{'OneFS'}) { return 0 } if (!defined $check) { return $CLUSTER_INFO{'OneFS'} } if (ref($check) eq 'Regexp') { return $CLUSTER_INFO{'OneFS'} =~ $check } if ($check =~ /^>=(\S+)/) { return compareVersions($CLUSTER_INFO{'OneFS'}, $1) >= 0; } if ($check =~ /^<=(\S+)/) { return compareVersions($CLUSTER_INFO{'OneFS'}, $1) <= 0; } if ($check =~ /^>(\S+)/) { return compareVersions($CLUSTER_INFO{'OneFS'}, $1) > 0; } if ($check =~ /^<(\S+)/) { return compareVersions($CLUSTER_INFO{'OneFS'}, $1) < 0; } return $CLUSTER_INFO{'OneFS'} eq $check; } # End isCurrentOneFS ### # isDestinationOneFS: Uses either Regexp or scalar to test destination OneFS sub isDestinationOneFS { my ($check) = @_; if (!$RPS{'Target'}) { return 0 } if (!defined $check) { return $RPS{'Target'} } if (ref($check) eq 'Regexp') { return $RPS{'Target'} =~ $check } if ($check =~ /^>=(\S+)/) { return compareVersions($RPS{'Target'}, $1) >= 0; } if ($check =~ /^<=(\S+)/) { return compareVersions($RPS{'Target'}, $1) <= 0; } if ($check =~ /^>(\S+)/) { return compareVersions($RPS{'Target'}, $1) > 0; } if ($check =~ /^<(\S+)/) { return compareVersions($RPS{'Target'}, $1) < 0; } return $RPS{'Target'} eq $check; } # End isDestinationOneFS sub isHardened { if (defined $CLUSTER_INFO{'Hardened'}) { return $CLUSTER_INFO{'Hardened'} } # Default to not hardened $CLUSTER_INFO{'Hardened'} = 0; # Gather hardening status my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'ls /etc/ifs/hardening_info.txt']; my $CMD_LOGS = q[2>&1 grep -H -m 1 . */etc.tar/etc/ifs/hardening_info.txt]; my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # If any nodes ard hardened, the cluster should be foreach my $lnn (@LNNS) { if ($gather->{$lnn}) { $CLUSTER_INFO{'Hardened'} = 1; last; } } # Return whether hardened return $CLUSTER_INFO{'Hardened'}; } ### # isLive: returns 1 when ran live on a cluster or 0 when in a logset sub isLive { # Previously called, return value if (defined $RPS{'Live'}) { return $RPS{'Live'} } # If /ifs exists, assume it is a live system if (-d '/ifs') { return $RPS{'Live'} = 1 } # If package_info.xml exists, assume it is a logset if (-e 'package_info.xml') { return $RPS{'Live'} = 0 } # If unable to determine where we are, exit printUrgent('Unable to determine where script is running, please run on a'. ' live cluster or in the root directory of a logset.'); exit 1; } # End isLive # isOneFSUpgrade: Returns true if the source->destination is a OneFS upgrade sub isOneFSUpgrade { my ($source, $destination) = @_; # If both arguments are missing if (!$source and !$destination) { $destination = isDestinationOneFS(); $source = isCurrentOneFS(); } # If either argument is missing if (!$source or !$destination) { return 0 } # If source is greater than or equal to the destination if (compareVersions($source, $destination) >= 0) { return 0 } # Split the code into major, minor, service pack, and patch my ($s_major, $s_minor, $s_service, $s_patch) = split(/\./, $source); my ($d_major, $d_minor, $d_service, $d_patch) = split(/\./, $destination); # If major / minor / service pack levels differ if ($s_major != $d_major or $s_minor != $d_minor or $s_service != $d_service) { return 1; } # If less than OneFS 9.1 if ($s_major < 9 and $s_minor < 1) { return 1 } # Else return 0; } # End isOneFSUpgrade sub isServiceDisabled { gatherServices(); return 1 if @_ and $_[0] and $CLUSTER_INFO{'Services'}{$_[0]} and $CLUSTER_INFO{'Services'}{$_[0]} =~ /disabled/i; return 0; } sub isServiceEnabled { gatherServices(); return 1 if @_ and $_[0] and $CLUSTER_INFO{'Services'}{$_[0]} and $CLUSTER_INFO{'Services'}{$_[0]} =~ /enabled/i; return 0; } sub isShortName { my ($hostname) = @_; return 1 if defined $hostname and $hostname !~ /\./; return 0; } sub isVirtual { gatherProductInfo(); return $CLUSTER_INFO{'Virtual'}; } ### # parseArguments: Parses command line arguments and sets global variables. sub parseArguments { ### # Parse CLI arguments my %arguments = (); while ($_ = shift) { if (/^\d+\.\d+\.\d+(\.\d+)?$/) { $arguments{'OneFS'}{'Destination'} = $_; } elsif (/^-?-a(?:utotc)?$/) { $arguments{'AutoTC'} = 1; } elsif (/^-?-d(ebug)?$/) { $arguments{'Debug'} = 1; } elsif (/^-?-d(?:rive-)?s(?:upport-)?p(?:ackage)?(?:=(.*))?$/) { $arguments{'Drive Firmware'}{'Destination'} = ($1)?$1:shift; } elsif (/^-?-d(?:isable-)?p(?:re-)?p(?:patched)?$/) { $arguments{'OneFS'}{'Standalone'} = 1; } elsif (/^-?-e(xtra)?$/) { $arguments{'Verbose'} = 1; } elsif (/^-?-h(elp)?$/) { $arguments{'Help'} = 1; } elsif (/^-?-j(son)?$/) { $arguments{'JSON'} = 1; } elsif (/^-?-l(?:ist)(?:=(.*))?$/) { $arguments{'CheckList'} = ($1)?$1:shift; } elsif (/^-?-p(?:riority-)?d(?:rive-)?f(?:irmware-)?o(?:nly)?$/) { $arguments{'Priority DF Only'} = 1; } elsif (/^-?-p(?:riority-)?n(?:ode-)?f(?:irmware-)?o(?:nly)?$/) { $arguments{'Priority NF Only'} = 1; } elsif (/^-?-r(?:un)?(?:=(.*))?$/) { push(@{$arguments{'Run'}},($1)?$1:shift); } elsif (/^-?-s(imultaneous)?$/) { $arguments{'Simultaneous'} = 1; $arguments{'OneFS'}{'Method'} = 'simultaneous'; $arguments{'Patch'}{'Method'} = 'simultaneous'; } elsif (/^-?-u(pgradeplan|pgradation)?$/) { $arguments{'UpgradePlan'} = 1; } elsif (/^-?-v(ersion)?$/) { $arguments{'Version'} = 1; } elsif (/^-?-f(co)?$/) { $arguments{'F010517EE'} = 1; } elsif (/^-?-f(co-)?r(olling)?$/) { $arguments{'F010517EE'} = 1; } elsif (/^-?-f010517ee$/) { $arguments{'F010517EE'} = 1; } elsif (/^-?-f121917ee$/) { $arguments{'F121917EE'} = 1; } elsif (/^-?-f011018ee$/) { $arguments{'F011018EE'} = 1; } elsif (/^-?-f033018ee$/) { $arguments{'F033018EE'} = 1; } elsif (/^-?-f082318ee$/) { $arguments{'F082318EE'} = 1; } elsif (/^-?-t102518ee$/) { $arguments{'T102518EE'} = 1; } elsif (/^-?-t062519ee$/) { $arguments{'T062519EE'} = 1; } elsif (/^-?-et003823$/) { $arguments{'ET003823'} = 1; } elsif (/^--definition$/) { $arguments{'Definition'} = 1; } elsif (/^-?-d(?:rive-)?f(?:irmware)?(?:=(.*))?$/) { $arguments{'Drive Firmware Options'} = ($1)?$1:shift; } elsif (/^-?-n(?:ode-)?f(?:irmware-?)?(?:p(?:ackage)?)?(?:=(.*))?$/) { $arguments{'Node Firmware Options'} = ($1)?$1:shift; } elsif (/^-?-o(?:nefs)?(?:=(.*))?$/) { $arguments{'OneFS Options'} = ($1)?$1:shift; } elsif (/^-?-p(?:atches)?(?:=(.*))?$/) { $arguments{'Patch Options'} = ($1)?$1:shift; } else { local *STDOUT = *STDERR; printCheck('Isilon On-Cluster Analysis', $RPS{'Version'}); print "\nUnrecognized command line argument provided: $_\n\n"; printUsage(); exit 1; } } ### # Validate arguments ### our(%NodeFirmware, %DriveFirmware, %DriveSupport, %Code_Matrix, %CheckLists, %Patch_Releases); # Define any undefined arguments as an empty string. foreach my $key (keys %arguments) { $arguments{$key} = '' if not defined $arguments{$key}; } # Set logging level if ($arguments{'Debug'}) { $Logger::LEVEL = &DEBUG; } elsif ($arguments{'Verbose'}) { $Logger::LEVEL = &VERBOSE; } # Prints the check definition if ($arguments{'Definition'}) { printDefinitionJSON(); exit; } # Enables JSON mode if ($arguments{'JSON'}) { *IOCA::printCheck = sub { }; *IOCA::printUrgent = sub { }; Logger::enableJSON(); $arguments{'Patch'}{'Destination'} = 'None'; } # Display version printCheck('Isilon On-Cluster Analysis', $RPS{'Version'}); # Exits after displaying version if ($arguments{'Version'}) { exit } # Displays help screen and exits if ($arguments{'Help'}) { printUsage(); exit; } # OneFS options if (defined $arguments{'OneFS Options'}) { $_ = $arguments{'OneFS Options'}; if (/([0-9.]+)/) { $arguments{'OneFS'}{'Destination'} = $1; } if (/simultaneous/i) { $arguments{'OneFS'}{'Method'} = 'simultaneous'; $arguments{'Simultaneous'} = 1; # FixMe } if (/rolling/i) { $arguments{'OneFS'}{'Method'} = 'rolling' } if (/parallel/i) { $arguments{'OneFS'}{'Method'} = 'parallel' } if (/standalone|dpp|disable-pre-patched/i) { $arguments{'OneFS'}{'Standalone'} = 1; } if (/exclude-nf/) { $arguments{'OneFS'}{'Exclude Node Firmware'} = 1 } } # Validates target OneFS is recognized if (defined $arguments{'OneFS'}{'Destination'}) { my @destinations = merge(keys %Code_Matrix, keys %Patch_Releases); @destinations = grep {$_ =~ /^\d+\.\d+\.\d+\.\d+$/} @destinations; @destinations = sort {compareVersions($b, $a)} @destinations; my @exact_match = grep {$_ eq $arguments{'OneFS'}{'Destination'}} @destinations; if (scalar(@exact_match) == 1) { $arguments{'OneFS'}{'Destination'} = $exact_match[0]; } else { my @results = grep {$_ =~ /^$arguments{'OneFS'}{'Destination'}/} @destinations; if (not @results) { print STDERR "\nRequested OneFS destination version, $arguments{'OneFS'}{'Destination'},". " was not recognized."; print STDERR "\nAvailable versions to check are: @destinations\n\n"; exit 1; } elsif (scalar(@results) > 1) { print STDERR "\nMultiple OneFS versions match $arguments{'OneFS'}{'Destination'}: @results"; print STDERR "\nAvailable versions to check are: @destinations\n\n"; exit 1; } else { $arguments{'OneFS'}{'Destination'} = $results[0]; } } } # Patch options if (defined $arguments{'Patch Options'}) { $_ = $arguments{'Patch Options'}; if (/none/i) { $arguments{'Patch'}{'Destination'} = 'None' } if (/simultaneous/i) { $arguments{'Patch'}{'Method'} = 'simultaneous' } if (/rolling/i) { $arguments{'Patch'}{'Method'} = 'rolling' } if (/parallel/i) { $arguments{'Patch'}{'Method'} = 'parallel' } } # Drive firmware options if (defined $arguments{'Drive Firmware Options'}) { $_ = $arguments{'Drive Firmware Options'}; if (/none/i) { $arguments{'Drive Firmware'}{'Destination'} = 'None'; } if (/([0-9.]+)/i) { $arguments{'Drive Firmware'}{'Destination'} = $1; } } # Validates target Drive Support Package is recognized if (defined $arguments{'Drive Firmware'}{'Destination'} and not grep {$_ eq $arguments{'Drive Firmware'}{'Destination'}} keys %DriveSupport) { print STDERR "\nRequested Drive Support Package version, $arguments{'Drive Firmware'}{'Destination'}, was not recognized."; print STDERR "\nAvailable versions to check are: ".join(', ',sort {compareVersions($b, $a)} keys %DriveSupport)."\n\n"; exit 1; } # Node firmware options if (defined $arguments{'Node Firmware Options'}) { $_ = $arguments{'Node Firmware Options'}; if (/([0-9.]+)/i) { $arguments{'Node Firmware'}{'Destination'} = $1; } if (/none/i) { $arguments{'Node Firmware'}{'Destination'} = 'None'; } if (/simultaneous/i) { $arguments{'Node Firmware'}{'Method'} = 'simultaneous' } if (/balanced|parallel/i) { $arguments{'Node Firmware'}{'Method'} = 'parallel' } if (/rolling/i) { $arguments{'Node Firmware'}{'Method'} = 'rolling' } } # Validates target Node Firmware Package is recognized if (defined $arguments{'Node Firmware'}{'Destination'} and not grep {$_ eq $arguments{'Node Firmware'}{'Destination'}} keys %NodeFirmware) { print STDERR "\nRequested Node Firmware Package version, $arguments{'Node Firmware'}{'Destination'}, was not recognized."; print STDERR "\nAvailable versions to check are: ".join(', ',sort {compareVersions($b, $a)} keys %NodeFirmware)."\n\n"; exit 1; } # Only allow checking for one FCO/TSE at a time if (1 < grep {defined} ($arguments{'T102518EE'}, $arguments{'F010517EE'}, $arguments{'F121917EE'}, $arguments{'F011018EE'}, $arguments{'F033018EE'}, $arguments{'F082318EE'}, $arguments{'T062519EE'}, $arguments{'ET003823'})) { # Only one FCO should be specified at any given time? print STDERR "\nMultiple FCO arguments provided, only one TSE/FCO should be handled at once.\n\n"; exit 1; } # Validates run argument is supported if (defined $arguments{'Run'}) { my @unrecognized = (); my @checks = (); foreach my $check (@{$arguments{'Run'}}) { if (defined $check and $CHECKS{$check}) { push(@checks, $check); } else { push(@unrecognized, $check); } } if (@unrecognized) { print STDERR "\nRequested checks, ".join(', ', @unrecognized).', was not recognized.'; print STDERR "\nAvailable checks are: "; foreach my $check (sort keys %CHECKS) { next unless $CHECKS{$check}{'Description'}; printf STDERR "\n\t%-30s%s", $check, $CHECKS{$check}{'Description'}; } print STDERR "\n\n"; exit 1; } if (@checks) { @{$arguments{'Run'}} = @checks; } else { delete $arguments{'Run'}; } } # Validates list argument is supported if (defined $arguments{'CheckList'} and not grep {$_ eq $arguments{'CheckList'}} keys %CheckLists) { print STDERR "\nRequested check list, $arguments{'CheckList'}, was not recognized."; print STDERR "\nAvailable check lists are: ".join(', ', sort keys %CheckLists); print STDERR "\n\n"; exit 1; } # F010517EE if ($arguments{'F010517EE'}) { if ($arguments{'Node Firmware'}{'Destination'} and compareVersions($arguments{'Node Firmware'}{'Destination'}, '9.3.5') < 0) { print STDERR "\nRequested Node Firmware Package version, $arguments{'Node Firmware'}{'Destination'}, is less than Node Firmware Package 9.3.5 which is required for FCO F010517EE.\n\n"; exit 1; } if ($arguments{'OneFS'}{'Destination'} and (($arguments{'OneFS'}{'Destination'} =~ /^7/ and compareVersions($arguments{'OneFS'}{'Destination'}, '7.2.1.6') < 0) or ($arguments{'OneFS'}{'Destination'} =~ /^8/ and compareVersions($arguments{'OneFS'}{'Destination'}, '8.0.0.4') < 0) or ($arguments{'OneFS'}{'Destination'} =~ /^8\.0\.1\.0/))) { print STDERR "\nDestination OneFS version, $arguments{'OneFS'}{'Destination'}, is less than the minimum required for FCO F010517EE: OneFS 8.0.0.4 or later, OneFS 8.0.1.1 or later, or OneFS 7.2.1.6 or later.\n\n"; exit 1; } $arguments{'UpgradePlan'} = 1; } # F121917EE if ($arguments{'F121917EE'}) { if ($arguments{'Node Firmware'}{'Destination'} and compareVersions($arguments{'Node Firmware'}{'Destination'}, '10.1.2') < 0) { print STDERR "\nDestination Node Firmware Package version, $arguments{'Node Firmware'}{'Destination'}, is less than Node Firmware Package 10.1.2 which is required for FCO F121917EE.\n\n"; exit 1; } if ($arguments{'OneFS'}{'Destination'}) { print STDERR "\nFCO F121917EE does not include a OneFS update, please re-run without a destination OneFS version.\n\n"; exit 1; } if ($arguments{'Drive Firmware'}{'Destination'}) { print STDERR "\nFCO F121917EE does not include drive firmware updates, please re-run without any drive firmware versions.\n\n"; exit 1; } $arguments{'UpgradePlan'} = 1; $arguments{'Drive Firmware'}{'Destination'} = 'None'; } # F011018EE if ($arguments{'F011018EE'}) { if ($arguments{'Node Firmware'}{'Destination'} and compareVersions($arguments{'Node Firmware'}{'Destination'}, '10.1.2') < 0) { print STDERR "\nDestination Node Firmware Package version, $arguments{'Node Firmware'}{'Destination'}, is less than Node Firmware Package 10.1.2 which is required for FCO F011018EE.\n\n"; exit 1; } if ($arguments{'OneFS'}{'Destination'}) { print STDERR "\nFCO F011018EE does not include a OneFS update, please re-run without a destination OneFS version.\n\n"; exit 1; } if ($arguments{'Drive Firmware'}{'Destination'}) { print STDERR "\nFCO F011018EE does not include drive firmware updates, please re-run without any drive firmware versions.\n\n"; exit 1; } $arguments{'UpgradePlan'} = 1; $arguments{'Drive Firmware'}{'Destination'} = 'None'; } # F033018EE if ($arguments{'F033018EE'}) { if ($arguments{'OneFS'}{'Destination'} and compareVersions($arguments{'OneFS'}{'Destination'}, '8.1.2.0') < 0) { print STDERR "\nFCO F033018EE includes a OneFS update to OneFS 8.1.2.0 and later, please re-run with a destination OneFS version of OneFS 8.1.2.0 or later.\n\n"; exit 1; } if ($arguments{'Drive Firmware'}{'Destination'} and compareVersions($arguments{'Drive Firmware'}{'Destination'}, '1.24') < 0) { print STDERR "\nFCO F033018EE includes drive firmware updates to Drive Support Package 1.24 and later, please re-run with a Drive Support Package of 1.24 or later.\n\n"; exit 1; } $arguments{'UpgradePlan'} = 1; } # F082318EE if ($arguments{'F082318EE'}) { if ($arguments{'OneFS'}{'Destination'} and compareVersions($arguments{'OneFS'}{'Destination'}, '8.1.0.4') < 0) { print STDERR "\nFCO F082318EE includes a OneFS update to OneFS 8.1.0.4 and later, please re-run with a destination OneFS version of OneFS 8.1.0.4 or later.\n\n"; exit 1; } if ($arguments{'Node Firmware'}{'Destination'} and compareVersions($arguments{'Node Firmware'}{'Destination'}, '10.1.6') < 0) { print STDERR "\nFCO F082318EE includes node firmware updates to Node Firmware Package 10.1.6 and later, please re-run with a Node Firmware Package of 10.1.6 or later.\n\n"; exit 1; } $arguments{'UpgradePlan'} = 1; } # T102518EE if ($arguments{'T102518EE'}) { if ($arguments{'OneFS'}{'Destination'} and compareVersions($arguments{'OneFS'}{'Destination'}, '8.1.0.4') < 0) { print STDERR "\nTSE T102518EE includes a OneFS update to OneFS 8.1.0.4 and later, please re-run with a destination OneFS version of OneFS 8.1.0.4 or later.\n\n"; exit 1; } if ($arguments{'Node Firmware'}{'Destination'} and compareVersions($arguments{'Node Firmware'}{'Destination'}, '10.2.1') < 0) { print STDERR "\nTSE T102518EE includes node firmware updates to Node Firmware Package 10.2.1 and later, please re-run with a Node Firmware Package of 10.2.1 or later.\n\n"; exit 1; } if ($arguments{'Drive Firmware'}{'Destination'} and compareVersions($arguments{'Drive Firmware'}{'Destination'}, '1.28') < 0) { print STDERR "\nTSE T102518EE includes drive firmware updates to Drive Support Package 1.28 and later, please re-run with a Drive Support Package of 1.28 or later.\n\n"; exit 1; } $arguments{'UpgradePlan'} = 1; } # T062519EE if ($arguments{'T062519EE'}) { $arguments{'UpgradePlan'} = 1; } # ET003823 if ($arguments{'ET003823'}) { $arguments{'UpgradePlan'} = 1; if ($arguments{'OneFS'}{'Destination'} and compareVersions($arguments{'OneFS'}{'Destination'}, '8.2.1.0') < 0) { print STDERR "\nPFN ET003823 includes a OneFS update to OneFS 8.2.1.0 build 004 from 8.2.1.0 build 003, please re-run with a destination OneFS version of OneFS 8.2.1.0 or later.\n\n"; exit 1; } } # If -u or a target OneFS code level is specified, run pre-upgrade checks if (!$arguments{'CheckList'} and ($arguments{'UpgradePlan'} or $arguments{'OneFS'}{'Destination'})) { $arguments{'CheckList'} = 'pre-upgrade'; } # Remove colored output for AutoTCs if ($arguments{'AutoTC'}) { disableColors(); no warnings 'redefine'; *colored = sub { return $_[0] if $_[0]; } } # Set legacy arguments if ($arguments{'Node Firmware'}{'Destination'}) { $arguments{'IsiFw'} = $arguments{'Node Firmware'}{'Destination'}; } if ($arguments{'OneFS'}{'Destination'}) { $arguments{'Target'} = $arguments{'OneFS'}{'Destination'}; } if ($arguments{'Patch'}{'Destination'}) { $arguments{'Patches'} = $arguments{'Patch'}{'Destination'}; if ($arguments{'Patch'}{'Destination'} eq 'None') { $arguments{'Disable Pre-Patched'} = 1; } } if ($arguments{'Drive Firmware'}{'Destination'}) { $arguments{'DSP'} = $arguments{'Drive Firmware'}{'Destination'}; } if ($arguments{'OneFS'}{'Standalone'}) { $arguments{'Disable Pre-Patched'} = 1; } # Store arguments foreach my $key (keys %arguments) { $RPS{$key} = $arguments{$key}; } } # End parseArguments ### # printDefinitionJSON: Prints the health check framework definition in JSON sub printDefinitionJSON { # Create an array of items which will be the definitions of each check my @items = (); my %exclude = ( 'checkLogLevel' => 1, ); # Add full IOCA as an item my %ioca = ( 'name' => "ioca", 'reference' => "https://www.dell.com/support/kbdoc/503265", 'summary' => "Runs IOCA health checks", 'description' => "", 'freshness' => 300, 'parameters' => [], 'unsupported' => []); push(@items, \%ioca); # Add each individual check as an item foreach my $check (sort keys %CHECKS) { next unless $CHECKS{$check}{'Description'}; if ($exclude{$check}) { next } my %item = (); $item{'name'} = "ioca_$check"; $item{'reference'} = ""; $item{'summary'} = $CHECKS{$check}{'Description'}; $item{'description'} = ""; $item{'freshness'} = 300; @{$item{'parameters'}} = (); @{$item{'unsupported'}} = (); push(@items, \%item); } # Create the definition hash my %definition = (); $definition{'version'} = $RPS{'Version'}; $definition{'node'} = "false"; $definition{'cost'} = 5; @{$definition{'items'}} = @items; # Print the definition print toJSON(\%definition)."\n"; } # printUsage: Displays help message sub printUsage { print <, --run= Executes only the specified check, can be included multiple times -u, --upgradeplan Includes an upgrade plan after health checks -o, --onefs Supports the following comma separated options [ex. 8.1.2,simultaneous]: Uses the provided destination OneFS version simultaneous Simultaneous OneFS upgrade parallel Parallel OneFS upgrade [requires 8.2.2+] rolling Rolling OneFS upgrade exclude-nf Upgrade plans will combine OneFS + node firmware by default [9.2 feature], this option disables that -p, --patches Supports the following comma separated options [ex. none,simultaneous]: none Opt out of patch recommendations simultaneous Simultaneous patch installs parallel Parallel patch installs [requires 9.1+] rolling Rolling patch installs -nf, --node-firmware Supports the following comma separated options [ex. 10.3.3,parallel]: Uses the provided version for node firmware checks none Opt out of node firmware recommendations simultaneous Simultaneous node firmware updates [requires 8.2+] parallel Parallel node firmware updates [requires 8.2+] rolling Rolling node firmware updates -df, --drive-firmware Supports the following comma separated options [ex. 1.32]: Uses the provided version for drive firmware checks none Opt out of drive firmware recommendations HELPMSG } # End printUsage ### # wasCalled: Returns 1 if the parent subroutine has already called # wasCalled, otherwise adds subroutine to list of called subroutines. sub wasCalled { my $subroutine = (caller(1))[3]; if (scalar(grep {$_ eq $subroutine} @{$RPS{'Called'}})) { return 1; } else { push(@{$RPS{'Called'}}, $subroutine); debug("--------------------"); debug(" Called: $subroutine"); debug("--------------------"); return 0; } } # End wasCalled $CHECKS{'checkIBInterfaces'}{'Description'} = "Checks for ib0/1 as being active, checks for ETA180317 IB switch firmware versions, and checks for overlapping IB networks."; sub checkIBInterfaces { return if wasCalled(); checkProcesses(); printCheck("IB Interfaces Active"); gatherNetworkInterfaces(); gatherArrayXML(); gatherETA180317(); my @nodes_with_issues = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); next unless $NODE_INFO{$lnn}{'Network'} and $CLUSTER_INFO{'Active IB Interfaces'}; verbose("ib0 Status: $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'}") if $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'}; verbose("ib1 Status: $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'}") if $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'}; if ($CLUSTER_INFO{'Active IB Interfaces'} == 1) { if (($NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} eq 'active') or ($NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} eq 'active')) { debug("checkIBInterfaces: Node $lnn passes IB interfaces active check"); } else { push(@nodes_with_issues, $lnn); debug("checkIBInterfaces: Node $lnn does not have at least one active IB interface"); } } elsif ($CLUSTER_INFO{'Active IB Interfaces'} == 2) { if (($NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib0'}{'status'} eq 'active') and ($NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} and $NODE_INFO{$lnn}{'Network'}{'ib1'}{'status'} eq 'active')) { debug("checkIBInterfaces: Node $lnn passes IB interfaces active check"); } else { push(@nodes_with_issues, $lnn); debug("checkIBInterfaces: Node $lnn does not have at least two active IB interface"); } } } # Check for overlapping IB networks if ($CLUSTER_INFO{'Interfaces'}) { debug("checkIBInterfaces: Searching for minimum netmask..."); my $min_netmask = 0b11111111111111111111111111111111; foreach my $interface (sort keys %{$CLUSTER_INFO{'Interfaces'}}) { verboseHeader("Internal Interface $interface"); verbose("Flags: $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'}; verbose("Low: $CLUSTER_INFO{'Interfaces'}{$interface}{'Low'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'Low'}; verbose("High: $CLUSTER_INFO{'Interfaces'}{$interface}{'High'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'High'}; verbose("Netmask: $CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}; verbose("MTU: $CLUSTER_INFO{'Interfaces'}{$interface}{'MTU'}") if $CLUSTER_INFO{'Interfaces'}{$interface}{'MTU'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface}{'Flags'} =~ /enabled/; my $bin_netmask = convertIPToNumeric($CLUSTER_INFO{'Interfaces'}{$interface}{'Netmask'}); $min_netmask = $bin_netmask if not defined $min_netmask or $bin_netmask < $min_netmask; } debug("checkIBInterfaces: Minimum mask: ".sprintf("%b", $min_netmask)); debug("checkIBInterfaces: Evaluating for overlap between IB networks..."); my @seen_interfaces = (); foreach my $interface_a (keys %{$CLUSTER_INFO{'Interfaces'}}) { next unless $CLUSTER_INFO{'Interfaces'}{$interface_a}{'Netmask'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_a}{'Flags'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_a}{'Flags'} =~ /enabled/; push(@seen_interfaces, $interface_a); my $a_lowaddr_bin = convertIPToNumeric($CLUSTER_INFO{'Interfaces'}{$interface_a}{'Low'}); foreach my $interface_b (keys %{$CLUSTER_INFO{'Interfaces'}}) { next if $interface_a eq $interface_b; next unless $CLUSTER_INFO{'Interfaces'}{$interface_b}{'Netmask'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_b}{'Flags'}; next unless $CLUSTER_INFO{'Interfaces'}{$interface_b}{'Flags'} =~ /enabled/; my @results = grep {$_ eq $interface_b} @seen_interfaces; next if @results; my $b_lowaddr_bin = convertIPToNumeric($CLUSTER_INFO{'Interfaces'}{$interface_b}{'Low'}); if (($a_lowaddr_bin & $min_netmask) == ($b_lowaddr_bin & $min_netmask)) { debug("checkIBInterfaces: Found overlap between $interface_a and $interface_b"); warning("Overlap found between IB interfaces $interface_a and $interface_b"); } else { debug("checkIBInterfaces: No overlap between $interface_a and $interface_b"); } } } } warning(@{$RPS{'ETA'}{'180317'}{'Message'}}) if $RPS{'ETA'}{'180317'}{'Message'}; fail("The following nodes do not have $CLUSTER_INFO{'Active IB Interfaces'} active IB interfaces: ".join(", ", @nodes_with_issues)) if @nodes_with_issues; info("Refer to KB515571 (https://www.dell.com/support/kbdoc/515571) for details.") if isResults(&WARN); printResults(); } # End checkIBInterfaces # checkInternalPingFail: Helper used by checkInternalPing sub checkInternalPingFail { my ($name, $network) = @_; if (!$name or !$network) { return } # Setup my $SPF_FAIL_NETWORK = 'Ping tests for the %s network failed for all'. ' nodes, the network may be down'; my $SPF_FAIL_INTERFACE = 'Ping tests for the %s network failed from the'. ' following nodes to all other nodes, the ports for this network on'. ' these nodes may be down: %s'; my $SPF_FAIL_PAIRS = 'Ping tests for the %s network failed between the'. ' following pairs of nodes: %s'; # Analyze my %fail = (); foreach my $lnn (@LNNS) { # No results for $lnn if (!$network->{$lnn}) { next } # No ping issues if ($network->{$lnn} =~ /^0+$/) { next } # Split each character into a result for a node my @results = split(//, $network->{$lnn}); # Check for any failed pings foreach my $i (0..$#LNNS) { # Skip checking pings to and from the same node if ($lnn eq $LNNS[$i]) { next } # results are 1 if ping failed, add it to the list if ($results[$i]) { push(@{$fail{$lnn}}, $LNNS[$i]) } } # Ping failed to all other nodes from $lnn if (scalar(@{$fail{$lnn}}) >= $#LNNS) { $fail{$lnn} = 'all' } } # The nodes where ping failed to all other nodes my @nodes = grep {$fail{$_} eq 'all'} keys %fail; # Flag the network is down when ping fails between all nodes if (scalar(@nodes) == scalar(@LNNS)) { fail(sprintf($SPF_FAIL_NETWORK, $name)); return; } # Flag nodes where the interface is down if (@nodes) { fail(sprintf($SPF_FAIL_INTERFACE, $name, compressRange(@nodes))); } # Generate pairs of nodes that failed ping tests my @pairs = (); foreach my $lnn (keys %fail) { # Skip nodes flagged with a down interface if ($fail{$lnn} eq 'all') { next } foreach my $remote_lnn (@{$fail{$lnn}}) { # Skip nodes flagged with a down interface if ($fail{$remote_lnn} eq 'all') { next } # Skip nodes already added to the pairs list in reverse order if (grep { $_ eq "$remote_lnn:$lnn" } @pairs) { next } # Add to the pairs of failed ping tests push(@pairs, "$lnn:$remote_lnn"); } } if (@pairs) { fail(sprintf($SPF_FAIL_PAIRS, $name, join(', ', @pairs))); } } # End checkInternalPingFail $CHECKS{'checkInternalPing'}{'Description'} = 'Checks internal network by performing network ping operations'; sub checkInternalPing { if (wasCalled()) { return } if (isCompliance()) { return unsupported('Not supported on compliance mode enabled clusters'); } if (!isLive()) { return unsupported('Not supported on log gathers'); } # Setup my $CMD_NODE = q[ret="";]. q[for i in $(isi_nodes %{internal_a},%{internal_b},%{failover}); do]. q[ v=0;]. q[ ping -o -c2 -t4 $(echo $i|cut -d, -f1)||((v+=1));]. q[ ping -o -c2 -t4 $(echo $i|cut -d, -f2)||((v+=2));]. q[ ping -o -c2 -t4 $(echo $i|cut -d, -f3)||((v+=4));]. q[ ret="$ret$v";]. q[done &>/dev/null;]. q[echo $ret]; my $CMD = qq[2>/dev/null isi_for_array '$CMD_NODE']; my $SPF_MALFORMED = 'Results from the following nodes may be incomplete: %s'; printCheck('Internal Network Ping'); # Gather my $gather = gatherCluster($CMD); my $expected_length = $CLUSTER_INFO{'Node Count'}; my @malformed = (); my %results = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next } my $line = pop(@{$gather->{$lnn}}); # Trim white space $line =~ s/^\s*|\s*$//g; # Each line must match the number of nodes in the cluster if (length($line) != $expected_length) { push(@malformed, $lnn); next; } # Extract results for int_[a]/int_[b]/[f]ailover $results{'a'}{$lnn} = $line & (1x$expected_length); $results{'b'}{$lnn} = $line & (2x$expected_length); $results{'f'}{$lnn} = $line & (4x$expected_length); $results{'b'}{$lnn} =~ s/2/1/g; $results{'f'}{$lnn} =~ s/4/1/g; } # Analyze & Flag checkInternalPingFail('int-a', $results{'a'}); checkInternalPingFail('int-b', $results{'b'}); checkInternalPingFail('failover', $results{'f'}); # Flag if (@malformed) { warning(sprintf($SPF_MALFORMED, compressRange(@malformed))); } # Output printResults(); } # End checkInternalPing $CHECKS{'checkSwitchCompatibility'}{'Description'} = 'Checks backend Dell'. ' switches to confirm they are at least version 10.5.0.6'; sub checkSwitchCompatibility { if (wasCalled()) { return } if (!hasDellSwitch()) { return unsupported('No Dell backend switches identified'); } printCheck('Back-end Dell Switch OS Compatibility'); # Setup my $RE_DELL = qr/(Dell\s+\S+)\s+-\s+v(\d+\.\d+\.\S+)\s/; my $MSG_WARN = q[It was detected that a back-end Dell switch is on a]. q[ version prior to 10.5. OneFS 9 and later requires Dell Networking]. q[ OS version 10.5.0.6 or later.]; my $MSG_UPGRADE = q[It was detected that a back-end Dell switch is running]. q[ a version prior to 10.5. The switch must be upgraded to 10.5.0.6 or]. q[ later prior to upgrading OneFS. A case should be raised]. q[ requesting an upgrade of the back-end Dell switches in preparation]. q[ for an OneFS upgrade]; # Analyze my $has_old_switch_os = 0; if (defined $CLUSTER_INFO{'Switches'}) { NETWORK: foreach my $switches (values %{$CLUSTER_INFO{'Switches'}}) { foreach my $switch (@$switches) { if ($switch !~ $RE_DELL) { next } my ($switch_type, $switch_os) = ($1, $2); if (compareVersions($switch_os, '10.5') < 0) { $has_old_switch_os = 1; last NETWORK; } } } } # Flag if ($has_old_switch_os) { if (isCurrentOneFS('<9') and isDestinationOneFS('>=9')) { fail($MSG_UPGRADE); } else { warning($MSG_WARN); } } # Output printResults(); } # End checkSwitchCompatibility # gatherBESwitch: Gather backend switch details sub gatherBESwitch { if (wasCalled()) { return } if (isVirtual()) { return } if (isCurrentOneFS('<8.1')) { return } if (isLive() and isCompliance()) { return } gatherNetworkInterfaces(); if ($CLUSTER_INFO{'Active IB Interfaces'}) { return } # Setup my $PIPE_GREP = q(|egrep -n '": \[\]?$|^[\[\{]'); my $CMD_LIVE = qq[2>&1 (${SUDO} isi_dump_fabric int-a;]. qq[ ${SUDO} isi_dump_fabric int-b)].$PIPE_GREP; my $CMD_LOGS = q[2>&1 cat $(ls -S1 */isi_dump_fabric|head -1)].$PIPE_GREP; my $RE_NETWORK = qr/^(\d+):[\[\{]$/; my $RE_SWITCH = qr/"(.*)": \[$/; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $network = ''; foreach my $line (@$gather) { # Use line number to identify which network if ($line =~ $RE_NETWORK) { if ($1 == 1) { $network = 'int-a'; debug('Gathering int-a network'); } elsif ($1 > 1) { $network = 'int-b'; debug('Gathering int-b network'); } } if (!$network) { next } # Extract switch details and append to list of switches if ($line !~ $RE_SWITCH) { next } my $switch = $1; @{$CLUSTER_INFO{'Switches'}{$network}} = merge(@{$CLUSTER_INFO{'Switches'}{$network}}, $switch); debug("CLUSTER_INFO->Switches->${network}=$switch"); } } # End gatherBESwitch # hasDellSwitch: Return true if Dell switches identified on backend network sub hasDellSwitch { if (defined $CLUSTER_INFO{'hasDellSwitch'}) { return $CLUSTER_INFO{'hasDellSwitch'}; } if (isVirtual()) { return $CLUSTER_INFO{'hasDellSwitch'} = 0; } if (isCurrentOneFS('<8.1')) { return $CLUSTER_INFO{'hasDellSwitch'} = 0; } gatherNetworkInterfaces(); if ($CLUSTER_INFO{'Active IB Interfaces'}) { return $CLUSTER_INFO{'hasDellSwitch'} = 0; } # Check backend switches gatherBESwitch(); if (defined $CLUSTER_INFO{'Switches'}) { foreach my $switches (values %{$CLUSTER_INFO{'Switches'}}) { my ($res) = grep {$_ =~ /Dell/} @$switches; if ($res) { return $CLUSTER_INFO{'hasDellSwitch'} = 1; } else { return $CLUSTER_INFO{'hasDellSwitch'} = 0; } } } # If we made it this far there was no switch details gathered if (isLive()) { # On live clusters, attempt to ping the virtual IPs my $CMD_A = qq[2>&1 ${SUDO} isi_for_array -Q]. q[ 'ping6 -nc1 fdfe:9042:c53d::5eff:fe00:101 && echo PASS']. q[ | grep PASS]; my $gather_a = gatherLocal($CMD_A); my ($res_a) = grep {$_ =~ /PASS/} @$gather_a; if ($res_a) { return $CLUSTER_INFO{'hasDellSwitch'} = 1; } my $CMD_B = qq[2>&1 ${SUDO} isi_for_array -Q]. q[ 'ping6 -nc1 fda7:e6ee:2e09::5eff:fe00:102 && echo PASS']. q[ | grep PASS]; my $gather_b = gatherLocal($CMD_B); my ($res_b) = grep {$_ =~ /PASS/} @$gather_b; if ($res_b) { return $CLUSTER_INFO{'hasDellSwitch'} = 1; } } else { # On log gathers check ndp records my $RE_VIP = '^(fdfe:9042:c53d::5eff:fe00:101|fda7:e6ee:2e09::5eff:fe00:102) '; my $CMD = "egrep -h '$RE_VIP' */ndp"; my $gather = gatherLocal($CMD); my ($res) = grep {$_ =~ /$RE_VIP/} @$gather; if ($res) { return $CLUSTER_INFO{'hasDellSwitch'} = 1; } } return $CLUSTER_INFO{'hasDellSwitch'} = 0; } # End hasDellSwitch $CHECKS{'checkAggregationMode'}{'Description'} = "Checks that the aggregation mode is not Legacy FEC mode for upgrades to OneFS 8+"; sub checkAggregationMode { if (wasCalled()) { return } return unsupported("Only supported for upgrades from OneFS 7 and earlier to OneFS 8 and later.") unless isCurrentOneFS('<8.0') and isDestinationOneFS('>=8.0'); printCheck("Legacy Aggregation Mode"); gatherFlexNet(); if (!$CLUSTER_INFO{'FlexNet'}{'Pools'}) { fail("Missing network pool information"); printResults(); return; } debug("checkAggregationMode: Checking for network pools configured with Legacy FEC Aggregation Modes"); my @network_pools_with_legacy_fec = (); foreach my $name (keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}) { # Check for Legacy FEC mode for the network pool next unless $CLUSTER_INFO{'FlexNet'}{'Pools'}{$name}{'Aggregation Mode'} and $CLUSTER_INFO{'FlexNet'}{'Pools'}{$name}{'Aggregation Mode'} =~ /legacy/i; # Check that the pool has any aggregate network interfaces next unless $CLUSTER_INFO{'FlexNet'}{'Pools'}{$name}{'Members'} and grep(/agg/, @{$CLUSTER_INFO{'FlexNet'}{'Pools'}{$name}{'Members'}}); # Both conditions met, flag as impacted debug("checkAggregationMode: !!! Network Pool $name uses Aggregation Mode $CLUSTER_INFO{'FlexNet'}{'Pools'}{$name}{'Aggregation Mode'}"); push(@network_pools_with_legacy_fec, $name); } if (@network_pools_with_legacy_fec) { warning("Legacy link aggregation in use on the following subnet/pools: ".join(", ", @network_pools_with_legacy_fec)); addCustomerInformation("Your cluster has at least one pool with \"legacy\" link aggregation in use. This mode is no longer supported starting in OneFS v8.0.0. As part of the upgrade the affected pool(s) will be switched to a compatible FEC mode.interface Please refer to the OneFS Upgrade Planning and Process Guide at support.emc.com. Affected subnets / pools: ".join(", ", @network_pools_with_legacy_fec)); } info("Refer to KB486561 (https://www.dell.com/support/kbdoc/486561) for details.") if isResults(&WARN); printResults(); } # End checkAggregationMode $CHECKS{'checkGatewayPriority'}{'Description'} = "Checks for subnets with duplicate gateway priorities."; sub checkGatewayPriority { return if wasCalled(); gatherFlexNet(); return unless $CLUSTER_INFO{'FlexNet'}{'Subnets'}; printCheck("Duplicate Gateway Priority"); debug("checkGatewayPriority: Creating hash of array for each subnet priority."); my %subnet_by_priority = (); verboseHeader("Gateway Priority"); verbose(sprintf("%-30s %-20s %-10s", "Subnet", "Gateway", "Priority")); foreach my $id (keys %{$CLUSTER_INFO{'FlexNet'}{'Subnets'}}) { my %subnet = %{$CLUSTER_INFO{'FlexNet'}{'Subnets'}{$id}}; next unless $subnet{'Gateway Priority'} and $subnet{'Gateway'}; next if $subnet{'Gateway'} eq "0.0.0.0"; push(@{$subnet_by_priority{$subnet{'Gateway Priority'}}}, $id); verbose(sprintf("%-30s %-20s %-10s", $id, $subnet{'Gateway'}, $subnet{'Gateway Priority'})); } debug("checkGatewayPriority: Checking for duplicate gateway priorities"); my $found_duplicate_gateway_priorities = 0; foreach my $priority (keys %subnet_by_priority) { debug("checkGatewayPriority: Gateway priority of $priority for subnets: ".join(", ", @{$subnet_by_priority{$priority}})); next unless (scalar(@{$subnet_by_priority{$priority}} > 1)); $found_duplicate_gateway_priorities = 1; warning("Duplicate gateway priority of $priority found for subnets: ".join(", ", @{$subnet_by_priority{$priority}})); debug("checkGatewayPriority: !!! Duplicate gateway priority found for gateway priority $priority !!!"); } addCustomerInformation("It was detected that multiple network subnets have the same gateway priority. This could result in unpredictable routing issues and lead to temporary data unavailability. See KB462780 (https://www.dell.com/support/kbdoc/462780) for more information and the Isilon External Network Connectivity Guide (https://support.emc.com/docu58740).") if $found_duplicate_gateway_priorities; info("Refer to KB485036 (https://www.dell.com/support/kbdoc/485036) for details") if isResults(&WARN); printResults(); } # End checkGatewayPriority $CHECKS{'checkKB519890'}{'Description'} = "Checks for a known issue with LAGG interfaces in LACP mode when running OneFS 8.0.0.6, 8.0.1.2, 8.1.0.2, and 8.1.1.1"; sub checkKB519890 { if (wasCalled()) { return } # Skip unless upgrading to or running an impacted version return unless isDestinationOneFS(qr/^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/) or (!isDestinationOneFS() and isCurrentOneFS(qr/^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/)); printCheck("KB519890"); gatherNetworkInterfaces(); # Analyze information for issues. my $has_lacp = 0; my @nodes_with_incomplete_lagg = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Network'}; my $has_incomplete_lagg = 0; my @lagg_interfaces = grep {$_ =~ /^lagg/} keys %{$NODE_INFO{$lnn}{'Network'}}; foreach my $lagg_interface (@lagg_interfaces) { next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'}; next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggproto'} =~ /lacp/; $has_lacp = 1; my $total_ports = 0; my $inactive_ports = 0; foreach my $laggport (@{$NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}}) { my $interface = ""; $interface = $1 if $laggport =~ /^(\S+)/; next unless $interface and $NODE_INFO{$lnn}{'Network'}{$interface}{'status'}; $total_ports++; next if $NODE_INFO{$lnn}{'Network'}{$interface}{'status'} eq "active"; $inactive_ports++; } $has_incomplete_lagg = 1 if $inactive_ports and $total_ports ne $inactive_ports; } push(@nodes_with_incomplete_lagg, $lnn) if $has_incomplete_lagg; } # Fail during pre-checks for upgrades targetting an affected version if ($has_lacp) { $RPS{'KB519890'} = 1 if isCurrentOneFS(qr/^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/) and !hasPatch('patch-(225079|226984)'); if (isDestinationOneFS(qr/^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/)) { addCustomerInformation(fail("In OneFS 8.0.0.6, 8.0.1.2, 8.1.0.2, and 8.1.1.1, when some of the ports of a lagg interface in LACP mode go down with other ports still active, OneFS treats the lagg interface as down with No Carrier status. If this lagg is the only interface for front end network, connection to the node from outside the cluster will be lost.")) if $has_lacp; } # If we didn't fail above, warn if the current OneFS version is impacted elsif (isCurrentOneFS(qr/^(8\.0\.0\.6|8\.0\.1\.2|8\.1\.0\.2|8\.1\.1\.1)$/)) { addCustomerInformation(warning("In OneFS 8.0.0.6, 8.0.1.2, 8.1.0.2, and 8.1.1.1, when some of the ports of a lagg interface in LACP mode go down with other ports still active, OneFS treats the lagg interface as down with No Carrier status. If this lagg is the only interface for front end network, connection to the node from outside the cluster will be lost.")) if $has_lacp; } } if (@nodes_with_incomplete_lagg) { if (isDestinationOneFS()) { fail("Not all ports of a lagg interface are active on the following nodes which would cause the lagg interface to be down following the OneFS upgrade to $RPS{'Target'}: ".compressRange(@nodes_with_incomplete_lagg)); } else { fail("Not all ports of a lagg interface are active on the following nodes which may cause the lagg interface to be down: ".compressRange(@nodes_with_incomplete_lagg)); } } printResults(); } # End checkKB519890 $CHECKS{'checkKB521890'}{'Description'} = "Checks for criteria outlined in KB 521890"; sub checkKB521890 { if (wasCalled()) { return } # Skip unless upgrading to or running an impacted version (8.0.0.[56], 8.1.0.[123], 8.1.1.[01]) return unless isDestinationOneFS(qr/^(8\.0\.0\.[56]|8.1.0.[123]|8\.1\.1\.[01])$/) or (!isDestinationOneFS() and isCurrentOneFS(qr/^(8\.0\.0\.[56]|8\.1\.0\.[123]|8\.1\.1\.[01])$/)); printCheck("KB 521890"); gatherNetworkInterfaces(); # Analyze information for issues. my $has_lagg = 0; my %nodes_with_ix_lagg = (); my %nodes_with_em_lagg = (); my %nodes_with_igb_lagg = (); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Network'}; # Issue exists in lagg interfaces my @lagg_interfaces = grep {$_ =~ /^lagg/} keys %{$NODE_INFO{$lnn}{'Network'}}; next unless @lagg_interfaces; # Flag for if the cluster has any lagg ports $has_lagg = 1; # For each lagg interface, check each lagg port for impacted interface types foreach my $lagg_interface (@lagg_interfaces) { next unless $NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}; my @laggports = @{$NODE_INFO{$lnn}{'Network'}{$lagg_interface}{'laggport'}}; my @ix_interfaces = grep {$_ =~ /^ix\d/} @laggports; $nodes_with_ix_lagg{$lnn} = 1 if @ix_interfaces; my @em_interfaces = grep {$_ =~ /^em\d/} @laggports; $nodes_with_em_lagg{$lnn} = 1 if @em_interfaces; my @igb_interfaces = grep {$_ =~ /^igb\d/} @laggports; $nodes_with_igb_lagg{$lnn} = 1 if @igb_interfaces; } } if ($has_lagg) { # There are laggs # Flag issues in destination OneFS versions if (isDestinationOneFS()) { # Upgrading OneFS if (isDestinationOneFS(qr/^(8\.0\.0\.[56]|8.1.0.[123]|8\.1\.1\.[01])$/)) { # Upgrading to an impacted version if (%nodes_with_ix_lagg) { # Has Gen 6 impacted interfaces fail("In the destination OneFS $RPS{'Target'}, the following nodes have a high risk of being impacted by KB 521890: ".compressRange(keys %nodes_with_ix_lagg)); } if (%nodes_with_em_lagg or %nodes_with_igb_lagg) { # Has other Gen impacted interfaces warning("In the destination OneFS $RPS{'Target'}, the following nodes have a very low risk of being impacted by KB 521890: ".compressRange(merge(keys %nodes_with_em_lagg, keys %nodes_with_igb_lagg))); } } if (isDestinationOneFS(qr/^(8.1.0.[123]|8\.1\.1\.[01])$/) and isCurrentOneFS('<8.1')) { # Upgrading to an impacted OneFS 8.1 code level, perhaps to add Gen6 nodes? addCustomerInformation(fail("After upgrading to OneFS $RPS{'Target'}, avoid configuring 10 GbE Intel network interfaces (ix interfaces) in link aggregation. These interfaces are most commonly seen in Generation 6 nodes. Please review the KB 521890 for details on an issue which impacts 10 GbE Intel network interfaces when used in link aggregation.")); } } # Flag issues in current OneFS versions if (isCurrentOneFS(qr/^(8\.0\.0\.[56]|8\.1\.0\.[123]|8\.1\.1\.[01])$/)) { # Running an impacted version if (isDestinationOneFS() and !isDestinationOneFS(qr/^(8\.0\.0\.[56]|8.1.0.[123]|8\.1\.1\.[01])$/) and %nodes_with_ix_lagg) { critical("In the current OneFS $CLUSTER_INFO{'OneFS'}, the following nodes have a high risk of being impacted by KB 521890: ".compressRange(keys %nodes_with_ix_lagg)); # Toggle a flag that will be used to recommend against node firmware or reboot activities $RPS{'KB521890'} = 1; } else { if (%nodes_with_ix_lagg) { # Has Gen 6 impacted interfaces fail("In the current OneFS $CLUSTER_INFO{'OneFS'}, the following nodes have a high risk of being impacted by KB 521890: ".compressRange(keys %nodes_with_ix_lagg)); # Toggle a flag that will be used to recommend against node firmware or reboot activities $RPS{'KB521890'} = 1; } if (%nodes_with_em_lagg or %nodes_with_igb_lagg) { # Has other Gen impacted interfaces warning("In the current OneFS $CLUSTER_INFO{'OneFS'}, the following nodes have a very low risk of being impacted by KB 521890: ".compressRange(merge(keys %nodes_with_em_lagg, keys %nodes_with_igb_lagg))); } if (!%nodes_with_ix_lagg and isCurrentOneFS(qr/^(8\.1\.0\.[123]|8\.1\.1\.[01])$/)) { # Has no impacted interfaces, but might be adding Generation 6 nodes addCustomerInformation(warning("If planning to add Generation 6 nodes, please first review the KB 521890 for details on an issue which impacts 10 GbE (ix) network interfaces on Generation 6 nodes when used in link aggregation.")); } } } } printResults(); } # End checkKB521890 ### # checkLACPSFP: Checks for LACP on cxgb interfaces for KB 489262 $CHECKS{'checkLACPSFP'}{'Description'} = "Checks for LACP on cxgb interfaces for KB 489262"; sub checkLACPSFP { if (wasCalled()) { return } if (!isDestinationOneFS(qr/^8\.0\.0\.[012345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/)) { return } gatherNetworkInterfaces(); printCheck("LACP over CXGB interfaces with TwinAx SFP"); debug("checkLACPSFP: Checking for network pools configured with LACP aggregation mode"); my @nodes_affected = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$lnn}{'Network'}; foreach my $interface (keys %{$NODE_INFO{$lnn}{'Network'}}) { next unless $interface; if ($interface =~ /cxgb/ and $NODE_INFO{$lnn}{'Network'}{$interface}{'media'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggdev'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'media'} =~ /twinax/i and $NODE_INFO{$lnn}{'Network'}{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggdev'}}{'laggproto'} and $NODE_INFO{$lnn}{'Network'}{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggdev'}}{'laggproto'} =~ /lacp/i) { debug("checkLACPSFP: !!! Network interface $interface on node $lnn may be impacted"); push(@nodes_affected, $lnn) unless grep {$_ eq $lnn} @nodes_affected; } if ($NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} and $NODE_INFO{$lnn}{'Network'}{$interface}{'laggproto'} =~ /lacp/i) { foreach my $cxgb_interface (grep {$_ =~ /^cxgb/} @{$NODE_INFO{$lnn}{'Network'}{$interface}{'laggport'}}) { next unless $cxgb_interface; if($NODE_INFO{$lnn}{'Network'}{$cxgb_interface}{'media'} and $NODE_INFO{$lnn}{'Network'}{$cxgb_interface}{'media'} =~ /twinax/i) { debug("checkLACPSFP: !!! Network interface $interface on node $lnn may be impacted"); push(@nodes_affected, $lnn) unless grep {$_ eq $lnn} @nodes_affected; } } } } } if (@nodes_affected) { # FAIL message fail("The following nodes were found as having Chelsio NICs using TwinAx SFP transceivers that are part of an aggregate network configured as LACP: ".compressRange(@nodes_affected)); fail("An issue exists in OneFS 8.x releases where communication over the associated aggregate network interfaces may fail to work after an upgrade to impacted OneFS 8.x versions"); fail("A fix for this issue is targeted for OneFS 8.0.0.6, OneFS 8.0.1.2, and OneFS 8.1.0.1."); fail("A directed availability patch is available upon request and approval of Isilon support SMEs, however, if the only connectivity to the cluster is via impacted interfaces, this may leave the cluster in an unreachable state until the patch has been applied."); fail("Isilon Certified SFP transceivers are also known to work, review the 'SFP transceivers for Generation 4 and Generation 5 nodes' section of the 'Isilon Supportability and Compatibility Guide'."); # Information appended to customer information addCustomerInformation ("TwinAx SFP, LAGG, & LACP", "The following nodes were found as having Chelsio NICs using TwinAx SFP transceivers that are part of an aggregate network configured as LACP: ".compressRange(@nodes_affected), "An issue exists in OneFS 8.x releases where communication over the associated aggregate network interfaces may fail to work after an upgrade to impacted OneFS 8.x versions", "A fix for this issue is targeted for OneFS 8.0.0.6, OneFS 8.0.1.2, and OneFS 8.1.0.1.", "A directed availability patch is available upon request and approval of Isilon support SMEs, however, if the only connectivity to the cluster is via impacted interfaces, this may leave the cluster in an unreachable state until the patch has been applied.", "Isilon Certified SFP transceivers are also known to work, review the 'SFP transceivers for Generation 4 and Generation 5 nodes' section of the 'Isilon Supportability and Compatibility Guide'."); } printResults(); } # End checkLACPSFP $CHECKS{'checkNetstat'}{'Description'} = "Checks connections counts for specific protocols via netstat"; sub checkNetstat { return if wasCalled(); printCheck("Netstat Connections"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'awk "]. q[\\$4~/\.(20|21)$/{FTP++}]. q[\\$4~/\.(22)$/{SSH++}]. q[\\$4~/\.(135|137|138|139|445)$/{SMB++}]. q[\\$4~/\.(80)$/{HTTP++}]. q[\\$4~/\.(443)$/{HTTPS++}]. q[\\$4~/\.(8080|8081)$/{WEBUI++}]. q[\\$4~/\.(2049)$/{NFS++}]. q[\\$4~/\.(9999|3260|3205|860)$/{ISCSI++}]. q[\\$4~/\.(8021|8020)$/{HDFS++}]. q[END{]. q[print \\"FTP:\\" FTP;]. q[print \\"SSH:\\" SSH;]. q[print \\"SMB:\\" SMB;]. q[print \\"HTTP:\\" HTTP;]. q[print \\"HTTPS:\\" HTTPS;]. q[print \\"WEBUI:\\" WEBUI;]. q[print \\"NFS:\\" NFS;]. q[print \\"ISCSI:\\" ISCSI;]. q[print \\"HDFS:\\" HDFS;]. q[}" <(netstat -lan|grep ESTABLISHED)']); } else { $gather = gatherCluster(qq[for i in */netstat;do echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(20|21)\$/) C++}END{print ":FTP:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(22)\$/) C++}END{print ":SSH:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(135|137|138|139|445)\$/) C++}END{print ":SMB:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(80)\$/) C++}END{print ":HTTP:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(443)\$/) C++}END{print ":HTTPS:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(8080|8081)\$/) C++}END{print ":WEBUI:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(2049)\$/) C++}END{print ":NFS:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(9999|3260|3205|860)\$/) C++}END{print ":ISCSI:" C}' \$i; echo -n \$i; awk '{if(\$6~/ESTABLISHED/ && \$4~/\\.(8021|8020)\$/) C++}END{print ":HDFS:" C}' \$i; done]); } # Analyze information for issues. my %high_connection_counts = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } my %connection_counts = (); foreach my $line (@{$gather->{$lnn}}) { next unless ($line =~ /\s*(\S+):(\d+)/); verbose("$1 Client Connections: $2"); $connection_counts{$1} = $2; } push(@{$high_connection_counts{'FTP'}}, $lnn) if ($connection_counts{'FTP'} and $connection_counts{'FTP'} >= 25); push(@{$high_connection_counts{'HDFS'}}, $lnn) if ($connection_counts{'HDFS'} and $connection_counts{'HDFS'} >= 25); push(@{$high_connection_counts{'SMB'}}, $lnn) if ($connection_counts{'SMB'} and $connection_counts{'SMB'} >= 750); push(@{$high_connection_counts{'NFS'}}, $lnn) if ($connection_counts{'NFS'} and $connection_counts{'NFS'} >= 750); push(@{$high_connection_counts{'SSH'}}, $lnn) if ($connection_counts{'SSH'} and $connection_counts{'SSH'} >= 5); push(@{$high_connection_counts{'WEBUI'}}, $lnn) if ($connection_counts{'WEBUI'} and $connection_counts{'WEBUI'} >= 5); push(@{$high_connection_counts{'HTTP'}}, $lnn) if ($connection_counts{'HTTP'} and $connection_counts{'HTTP'} >= 250); push(@{$high_connection_counts{'ISCSI'}}, $lnn) if ($connection_counts{'ISCSI'} and $connection_counts{'ISCSI'} >= 2); push(@{$high_connection_counts{'HTTPS'}}, $lnn) if ($connection_counts{'HTTPS'} and $connection_counts{'HTTPS'} >= 10); } foreach my $protocol (keys %high_connection_counts) { warning("High connection count for $protocol on nodes: ".compressRange(@{$high_connection_counts{$protocol}})); } # Output Check result printResults(); } # End checkNetstat $CHECKS{'checkRoutingTables'}{'Description'} = "Displays routing table for each node"; sub checkRoutingTables { return if wasCalled(); printCheck("Routing Tables"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array netstat -rn]); } else { $gather = gatherCluster(q[for i in */netstat; do grep -H . $i | awk '/Routing tables/, /Active/'; done]); } # Analyze information for issues. foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { last if $line =~ /Active Internet connections/; verbose("$line"); } } # Output Check result printResults(); } # End checkRoutingTables sub gatherFlexNet { if (wasCalled()) { return } # The awk command in awk_string will remove xml groups we have no interest in my $awk_string = q[awk 'BEGIN{FS="[^0-9]*"}/device-id/{D=$2;next}/class-index/{I=$2;next}/<\/member>/{print "member:"D":"I;next}/<(ipranges|members|interfaces|provisioning-rules)>/{flag=1;next}/<\/(ipranges|members|interfaces|provisioning-rules)>/{flag=0;next}!flag{print}//{print}']; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { my ($file) = gatherLocal(q[2>/dev/null ls -t /etc/ifs/flx_config.xml /etc/ifs/flexnet/flx_config.xml /ifs/.ifsvar/modules/flexnet/flx_config.xml|head -1]); $gather = gatherLocal(qq[2>&1 $awk_string '$file']) if $file; } else { my ($file) = gatherLocal(q[2>/dev/null ls -t local/ifsvar_modules.tar/modules/flexnet/flx_config.xml */etcifs.tar/ifs/flx_config.xml */etc.tar/etc/ifs/flx_config.xml */etc.tar/etc/ifs/flexnet/flx_config.xml|head -1]); $gather = gatherLocal(qq[2>&1 $awk_string '$file']) if $file; } # Analyze information for issues. my @level = (); my @names = (); my %level_conv = ("groupnet","Groupnets","subnet","Subnets","network-pool","Pools"); my %tmp = (); my $gather_ssips = 0; my $tag = ""; foreach (@$gather) { $tag = $level[$#level] if !$tag and @level; last if /<\/flexnet/; if (/<(groupnet|subnet|network-pool)>/) { $tag = $1; push(@level, $tag); } elsif (/<\/(groupnet|subnet|network-pool)>/) { if ($tag ne $1) { return; } $CLUSTER_INFO{'FlexNet'}{$level_conv{$tag}}{join(".",@names)} = \%{$tmp{$tag}}; delete $tmp{$tag}; %{$tmp{$tag}} = (); pop(@level); pop(@names); $tag = ""; } elsif (/(\d+)(\d)([^<]+)/) { $gather_ssips = 1; } elsif (/<\/sc-service-addresses>/) { $gather_ssips = 0; } elsif ($gather_ssips) { if (/low>([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)([^<]+)(\d+):([^<]+)([^<]+)([^<]+)FlexNet=".toJSON($CLUSTER_INFO{'FlexNet'})); } ### # gatherNetworkInterfaces: Gathers network interface information. sub gatherNetworkInterfaces { return if wasCalled(); # Gather IP Addresses from ifconfig my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array ifconfig"); } else { $gather = gatherCluster("2>&1 grep -H . */ifconfig"); } # Store IP Addresses under NODE_INFO{$node}{'Network'}{$interface}{'addresses'} and # the status of each interface in NODE_INFO{$node}{'Network'}{$interface}{'status'} foreach my $node (@LNNS) { if (!$gather->{$node}) { missing($node); next; } my $interface = ""; foreach my $line (@{$gather->{$node}}) { $interface = $1 if ($line =~ /^\s*(\S+): flags/); next if (!$interface); # if ($line =~ /mtu (\d+)/) { # $NODE_INFO{$node}{'Network'}{$interface}{'mtu'} = $1; # debug("NODE_INFO->${node}->Network->${interface}->mtu=$1"); # } # if ($line =~ /options=(.*)/) { # $NODE_INFO{$node}{'Network'}{$interface}{'options'} = $1; # debug("NODE_INFO->${node}->Network->${interface}->options=$1"); # } if ($line =~ /lagg:\s+laggdev\s+(\S+)/) { $NODE_INFO{$node}{'Network'}{$interface}{'laggdev'} = $1; debug("NODE_INFO->${node}->Network->${interface}->laggdev=$1"); } if ($line =~ /media: (.*)/) { $NODE_INFO{$node}{'Network'}{$interface}{'media'} = $1; debug("NODE_INFO->${node}->Network->${interface}->media=$1"); } if ($line =~ /status: (.*)/) { $NODE_INFO{$node}{'Network'}{$interface}{'status'} = $1; debug("NODE_INFO->${node}->Network->${interface}->status=$1"); } if ($line =~ /inet (\S+)/) { push(@{$NODE_INFO{$node}{'Network'}{$interface}{'addresses'}}, $1); debug("NODE_INFO->${node}->Network->${interface}->addresses=$1"); } if ($line =~ /inet6 (\S+)/) { push(@{$NODE_INFO{$node}{'Network'}{$interface}{'addresses'}}, $1); debug("NODE_INFO->${node}->Network->${interface}->addresses=$1"); } if ($line =~ /^\s*laggproto\s+(\S+.*)$/) { $NODE_INFO{$node}{'Network'}{$interface}{'laggproto'} = $1; debug("NODE_INFO->${node}->Network->${interface}->laggproto=$1"); } if ($line =~ /^\s*laggport:\s+(\S.*)$/) { push(@{$NODE_INFO{$node}{'Network'}{$interface}{'laggport'}}, $1); debug("NODE_INFO->${node}->Network->${interface}->laggport=$1"); } # if ($line =~ /lladdr\s+(\S+)$/) { # $NODE_INFO{$node}{'Network'}{$interface}{'lladdr'} = $1; # debug("NODE_INFO->${node}->Network->${interface}->lladdr=$1"); # } } # Track nodes with BXE interfaces for BXE related patches push(@{$CLUSTER_INFO{'BXE Nodes'}}, $node) if ($NODE_INFO{$node}{'Network'}{'bxe0'} or $NODE_INFO{$node}{'Network'}{'bxe1'}); # Track nodes with CXGB interfaces push(@{$CLUSTER_INFO{'CXGB Nodes'}}, $node) if ($NODE_INFO{$node}{'Network'}{'cxgb0'} or $NODE_INFO{$node}{'Network'}{'cxgb1'}); # Tally how many active InfiniBand interfaces there is on each node, store this in CLUSTER_INFO{'Active IB Interfaces'} my $ibinterfaces = 0; $ibinterfaces++ if ($NODE_INFO{$node}{'Network'}{'ib0'} and $NODE_INFO{$node}{'Network'}{'ib0'}{'status'} eq 'active'); $ibinterfaces++ if ($NODE_INFO{$node}{'Network'}{'ib1'} and $NODE_INFO{$node}{'Network'}{'ib1'}{'status'} eq 'active'); if (!$CLUSTER_INFO{'Active IB Interfaces'} or $ibinterfaces > $CLUSTER_INFO{'Active IB Interfaces'}) { $CLUSTER_INFO{'Active IB Interfaces'} = $ibinterfaces; debug("CLUSTER_INFO->Active IB Interfaces=$ibinterfaces"); } } } # End gatherNetworkInterfaces sub checkKB535922 { if (wasCalled()) { return } if (isCurrentOneFS('8.2.0.0')) { return } if (!isDestinationOneFS('8.2.0.0')) { return } gatherFlexNet(); if (!$CLUSTER_INFO{'FlexNet'}{'Subnets'}) { return } # Setup my $MSG_SSIP = 'Immediately following the commit of the upgrade to OneFS'. ' 8.2.0.0, the SmartConnect Service IP addresses associated with'. ' subnets that have VLAN tagging may disappear. To avoid this, upgrade'. ' to a pre-patched OneFS 8.2 image or OneFS 8.2.1 or later release.'. ' For more details on this issue please review KB 535922'. ' [https://www.dell.com/support/kbdoc/535922 - {ISILON} Smartconnect'. ' Service IP Address (SSIP) goes missing on all configured VLAN'. ' interfaces after committing OneFS 8.2 upgrade]'; # Analyze and Flag foreach my $href_subnet (values %{$CLUSTER_INFO{'FlexNet'}{'Subnets'}}) { if ($href_subnet->{'VLAN Enabled'} and $href_subnet->{'VLAN Tag'}) { addCustomerInformation('SmartConnect', critical($MSG_SSIP)); return; } } } $CHECKS{'checkNetworkParallelUpgrade'}{'Description'} = 'Checks for the'. ' risk of inaccessible network pools during a parallel upgrade'; sub checkNetworkParallelUpgrade { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return unsupported('Unsupported prior to OneFS 8'); } # Setup my $MSG_MISSING_NETWORK = 'Missing network pool information'; my $MSG_MISSING_DISK = 'Missing disk pool information'; my $MSG_SINGLE_NEIGHBORHOOD = 'Clusters with multiple disk pools may benefit from parallel upgrades'. ' and this cluster has only one'; my $SPF_RISK = 'A parallel upgrade runs the risk of making one or more'. ' external networks temporarily unavailable. Affected pools: %s'; my $MIN_NODES = 2; printCheck('Network Pools - Parallel Upgrade Support'); gatherNeighborhoods(); if (!$CLUSTER_INFO{'Neighborhoods'}) { fail($MSG_MISSING_DISK); printResults(); return; } # Print neighborhoods as extra verboseHeader('Node Neighborhoods'); my $index = 1; foreach my $neighborhood (values %{$CLUSTER_INFO{'Neighborhoods'}}) { verbose("$index: [ @$neighborhood ]"); $index++; } gatherFlexNet(); if (!$CLUSTER_INFO{'FlexNet'}{'Pools'}) { fail($MSG_MISSING_NETWORK); printResults(); return; } if (scalar(keys %{$CLUSTER_INFO{'Neighborhoods'}}) <= 1) { info($MSG_SINGLE_NEIGHBORHOOD); printResults(); return; } # Analyze my %at_risk_subnets = (); foreach my $id (keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}) { my $href_pool = $CLUSTER_INFO{'FlexNet'}{'Pools'}{$id}; if (!$href_pool) { next } # Skip if pool is for Superna Eyeglass Runbook Robot if (defined $href_pool->{'Name'} and $href_pool->{'Name'} =~ /^eyeglass-robot/) { debug("Pool $id: Skipping Superna Eyeglass Runbook Robot pools"); next; } # Skip if pool has no members if (!defined $href_pool->{'Members'}) { debug("Pool $id: Skipping pools with no members"); next; } # Using the pool members, compile a list of nodes my @nodes = merge(map {$1 if /^(\d+):/} @{$href_pool->{'Members'}}); debug("Subnet: $id, Nodes: @nodes"); # Skip if pool has less than two nodes if (scalar(@nodes) < 2) { debug("Pool $id: Skipping pools with less than two nodes"); next; } # Check each neighborhood my $max_nodes = 0; foreach my $neighborhood (values %{$CLUSTER_INFO{'Neighborhoods'}}) { debug("Neighborhood: @$neighborhood"); my @pool_neighbors = array_duplicates(@nodes, @$neighborhood); debug("Neighbors in pool $id: @pool_neighbors"); my $num_nodes = scalar(@pool_neighbors); if ($num_nodes > $max_nodes) { $max_nodes = $num_nodes } } if ($max_nodes < $MIN_NODES) { $at_risk_subnets{$id} = 1 } } # Flag if (%at_risk_subnets) { my $msg = sprintf($SPF_RISK, join(', ', sort keys %at_risk_subnets)); if ((exists $RPS{'OneFS'}{'Method'} and $RPS{'OneFS'}{'Method'} eq 'parallel') or (exists $RPS{'Patch'}{'Method'} and $RPS{'Patch'}{'Method'} eq 'parallel') or (exists $RPS{'Node Firmware'}{'Method'} and $RPS{'Node Firmware'}{'Method'} eq 'parallel')) { warning($msg); } else { info($msg); } } printResults(); } # End checkNetworkParallelUpgrade # checkSCShortNames: Identify if any SC Zone Names or SC DNS Zone Aliases use a shortname, called by checkSmartConnect. sub checkSCShortNames { if (wasCalled()) { return } gatherFlexNet(); if (!$CLUSTER_INFO{'FlexNet'}{'Pools'}) { return } my %short_names = (); my @pids = sort keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}; foreach my $pid (@pids) { my $href_pool = $CLUSTER_INFO{'FlexNet'}{'Pools'}{$pid}; if (isShortName($href_pool->{'SC Zone Name'})) { push(@{$short_names{$pid}}, $href_pool->{'SC Zone Name'}); } if (!$href_pool->{'SC Zone Aliases'}) { next } foreach my $alias (@{$href_pool->{'SC Zone Aliases'}}) { if (isShortName($alias)) { push(@{$short_names{$pid}}, $alias); } } } if (!%short_names) { return } verboseHeader('SmartConnect Short Names'); foreach (sort keys %short_names) { verbose("Pool: $_, DNS Short Names: ".join(', ', @{$short_names{$_}})); } if ($RPS{'Target'} and isCurrentOneFS('<8.2') and compareVersions($RPS{'Target'}, '8.2.0') >= 0) { addCustomerInformation('SmartConnect', fail("Found SC Zone Name(s) and/or SC DNS Zone Alias(es) that are configured to use a DNS short name as opposed to a fully qualified domain name, this is not supported and will stop working after the upgrade to OneFS $RPS{'Target'}. For details on why it will no longer work in OneFS 8.2 or later see KB 534586 [https://www.dell.com/support/kbdoc/534586 - OneFS 8.2 if using (short name/Literal) instead of FQDN smartconnect refuses any queries]")); } else { warning("Found SC Zone Name(s) and/or SC DNS Zone Alias(es) that are configured to use a DNS short name as opposed to a fully qualified domain name, this is not supported and will stop working in OneFS 8.2 or later. For details on why it will no longer work in OneFS 8.2 or later see KB 534586 [https://www.dell.com/support/kbdoc/534586 - OneFS 8.2 if using (short name/Literal) instead of FQDN smartconnect refuses any queries]"); } } # End checkSCShortNames $CHECKS{'checkSmartConnect'}{'Description'} = 'Check that the SmartConnect Service IPs are all assigned and not being used for client connectivity.'; sub checkSmartConnect { if (wasCalled()) { return } gatherFlexNet(); my $href_subnets = $CLUSTER_INFO{'FlexNet'}{'Subnets'}; if (!$href_subnets) { return } my $href_pools = $CLUSTER_INFO{'FlexNet'}{'Pools'}; if (!$href_pools) { return } printCheck('SmartConnect Service IP'); gatherNetworkInterfaces(); # Run other SmartConnect related checks checkSSIPConnections(); checkKB535922(); checkSCShortNames(); # Check that each SSIP is assigned and not assigned more than once debug('checkSmartConnect: Looking for each SSIP address...'); verboseHeader('SmartConnect Service IP Locations'); my $spf = '%-20s %-5s %-10s'; verbose(sprintf($spf, 'SSIP', 'LNN', 'Network Interface')); verbose(sprintf($spf, '-'x20, '-'x5, '-'x10)); my %scip_counts = (); foreach my $subnet (keys %{$href_subnets}) { debug("checkSmartConnect: Investigating Subnet $subnet"); my $href_subnet = $href_subnets->{$subnet}; if (!$href_subnet->{'SSIP'}) { next } debug("checkSmartConnect: SSIPs: [ @{$href_subnet->{'SSIP'}} ]"); my ($zero_ssip) = grep {$_ eq '0.0.0.0'} @{$href_subnet->{'SSIP'}}; if ($zero_ssip) { next } my @pools = grep {$_ =~ /^$subnet[\.:]/} keys %{$href_pools}; if (!@pools) { next } debug("checkSmartConnect: Network Pools: [ @pools ]"); my $members = 0; foreach my $pool (@pools) { my $href_pool = $href_pools->{$pool}; if (!$href_pool->{'Members'}) { next } $members += scalar(@{$href_pool->{'Members'}}); } debug("checkSmartConnect: Subnet $subnet has $members members"); # Skip if no members if (!$members) { next } SCIP: foreach my $scip (@{$href_subnet->{'SSIP'}}) { debug("checkSmartConnect: Searching for SCIP: $scip"); $scip_counts{$scip} = 0; foreach my $lnn (@LNNS) { my $href_network = $NODE_INFO{$lnn}{'Network'}; if (!$href_network) { next } foreach my $interface (keys %{$href_network}) { my $href_interface = $href_network->{$interface}; if (!$href_interface) { next } foreach my $address (@{$href_interface->{'addresses'}}) { if ($address ne $scip) { next } verbose(sprintf($spf, $scip, $lnn, $interface)); $scip_counts{$scip}++; } } } } } # Flag any missing or repeated SSIPs my @missing_scips = grep {$scip_counts{$_} == 0} keys %scip_counts; if (@missing_scips) { fail("The following SmartConnect Service IP Address(es) could not be located on any node: @missing_scips") } my @repeat_scips = grep {$scip_counts{$_} > 1} keys %scip_counts; if (@repeat_scips) { fail("The following SmartConnect Service IP Address(es) were found on multiple nodes and may be encountering KB539139 [https://www.dell.com/support/kbdoc/539139 - Isilon: Smartconnect Service IP address (SSIP) may reside on multiple nodes after 8.2 upgrade]: @repeat_scips") } # Print results if (isResults(&WARN)) { info('Refer to KB 335093 (https://www.dell.com/support/kbdoc/335093) for details.') } printResults(); } # End checkSmartConnect # checkSSIPConnectionss: Checks if clients are connected to a SmartConnect # Service IP address (SSIPs). Called by checkSmartConnect. sub checkSSIPConnections { if (wasCalled()) { return } # Create a list of the SSIPs gatherFlexNet(); if (!$CLUSTER_INFO{'FlexNet'}{'Subnets'}) { return } my @SSIPs = (); foreach my $href_subnet (values %{$CLUSTER_INFO{'FlexNet'}{'Subnets'}}) { if (!$href_subnet->{'SSIP'}) { next } @SSIPs = merge(@SSIPs, @{$href_subnet->{'SSIP'}}); } # Return if there are no SSIPs if (!@SSIPs) { return } # Create a grep to capture clients that are connected to the SSIPs my $GREP_SSIP = sprintf('grep -E " (%s)."', join('|', @SSIPs)); $GREP_SSIP =~ s/\./\\./g; # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array]. qq[ 'netstat -lan | grep ESTABLISHED | $GREP_SSIP']; my $CMD_LOGS = qq[grep -H -a ESTABLISHED */netstat | $GREP_SSIP]; my $SPF = '%5s %-30s %-30s'; my $RE_PROTOCOLS = qr/\.(?:20|21|22|135|137|138|445|80|443|8080|8081|2049|8021|8020)$/; my $MSG_FAIL = 'Found client(s) connected to a SmartConnect Service IP'. ' address (SSIP). This is not supported and will stop working after'. ' upgrading to OneFS 8.2 or later. For details on why it will no'. ' longer work in OneFS 8.2 or later see KB 536650'. ' [https://www.dell.com/support/kbdoc/536650 - OneFS 8.2: Smartconnect'. ' Service IP Address (SSIP/VIP) does not allow connections beyond DNS'. ' traffic.]'; my $MSG_WARN = 'Found client(s) connected to a SmartConnect Service IP'. ' address (SSIP). This is not supported and will stop working in OneFS'. ' 8.2 or later. For details on why it will no longer work in OneFS 8.2'. ' or later see KB 536650 [https://www.dell.com/support/kbdoc/536650 -'. ' OneFS 8.2: Smartconnect Service IP Address (SSIP/VIP) does not allow'. ' connections beyond DNS traffic.]'; # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $found_clients_using_ssip = 0; verboseHeader('Clients Connected to SSIPs'); verbose(sprintf($SPF, 'LNN', 'Local Address', 'Remote Address')); verbose(sprintf($SPF, '-'x5, '-'x30, '-'x30)); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach my $line (@{$gather->{$lnn}}) { if ($line !~ /\s(\S+)\s+(\S+)\s+ESTABLISHED$/) { next } my ($local_addr, $remote_addr) = ($1, $2); if ($local_addr !~ $RE_PROTOCOLS) { next } $local_addr =~ s/\.(\d+)$/:$1/; $remote_addr =~ s/\.(\d+)$/:$1/; $found_clients_using_ssip = 1; verbose(sprintf($SPF, $lnn, $local_addr, $remote_addr)); } } # Flag if ($found_clients_using_ssip) { if (isDestinationOneFS('>=8.2') and isCurrentOneFS('<8.2')) { addCustomerInformation('SmartConnect', fail($MSG_FAIL)); } else { warning($MSG_WARN); } } } # End checkSSIPConnections $CHECKS{'checkKB489473'}{'Description'} = "Checks if any node meets criteria for KB489473"; sub checkKB489473 { return if wasCalled(); return unsupported("Unsupported on virtual platforms") if isVirtual(); return unsupported("Cluster contains no Generation 4 or Generation 5 nodes") unless $CLUSTER_INFO{'Gen5'} or $CLUSTER_INFO{'Gen4'}; printCheck("KB489473"); gatherNodeFirmware(); foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'Firmware'}; foreach my $device (keys %{$NODE_INFO{$lnn}{'Firmware'}}) { my $firmware = $NODE_INFO{$lnn}{'Firmware'}{$device}; next unless $device eq 'IsilonFPV1' and $firmware eq 'None'; push(@{$RPS{'Node Firmware'}{'489473'}{'Nodes'}}, $lnn); } } if ($RPS{'Node Firmware'}{'489473'}{'Nodes'}) { fail("The IsilonFPV1 device is reporting None on the following nodes: @{$RPS{'Node Firmware'}{'489473'}{'Nodes'}}"); } # Output Check result info("Refer to KB489473 (https://www.dell.com/support/kbdoc/489473) for details.") if isResults(&INFO); printResults(); } $CHECKS{'checkKB516613'}{'Description'} = "Checks if any node meets criteria for KB516613"; sub checkKB516613 { return if wasCalled(); gatherProductInfo(); return unsupported("Unsupported on virtual platforms") if isVirtual(); return unsupported("Cluster contains no Generation 5 nodes") unless $CLUSTER_INFO{'Gen5'}; printCheck("KB516613"); gatherNodeFirmware(); # Gather information from the cluster, not available via logs my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster (qq[ 2>&1 ${SUDO} isi_for_array -n ]. compressRange(@{$CLUSTER_INFO{'Gen5'}}). qq[ '/sbin/nvmecontrol logpage -p 3 lnv0 | grep "Slot"' ]); } # Analyze information for issues. my %nodes_with_slot_3_active = (); my %nodes_with_old_lox = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); next unless $NODE_INFO{$lnn}{'Generation'}; next unless $NODE_INFO{$lnn}{'Generation'} eq 'Gen5'; # Check LOx NVRAM card firmware version being impacted if ($NODE_INFO{$lnn}{'Firmware'}) { foreach my $device (keys %{$NODE_INFO{$lnn}{'Firmware'}}) { next unless $device =~ /LOx/; next unless $NODE_INFO{$lnn}{'Firmware'}{$device} =~ /rp180[abc]/; $nodes_with_old_lox{$lnn} = 1; } } # Gathered details required from here next unless $gather->{$lnn}; # Check if Slot 3 is active with an impacted firmware version foreach my $line (@{$gather->{$lnn}}) { verbose($line); next unless $line =~ /Slot 3:.*Active.*?rp180[abc]/; $nodes_with_slot_3_active{$lnn} = 1; } } # Flag the issues if (%nodes_with_old_lox) { warning("LOx NVRAM card running older rpc180[abc] firmware version on the following nodes: ".compressRange(keys %nodes_with_old_lox)); } if (%nodes_with_slot_3_active) { $RPS{'KB516613'} = 1; critical("KB 516613 criteria met on the following nodes: ".compressRange(keys %nodes_with_slot_3_active)); } if (!$RPS{'Live'} and %nodes_with_old_lox) { $RPS{'KB516613'} = 1; critical("KB 516613 criteria cannot be verified in logs, it will be assumed criteria is met."); } if ($RPS{'KB516613'}) { critical("Nodes that meet criteria for KB 516613 may fail to boot and require on-site remediation following KB 509549."); warning("Try to avoid any reboots before applying node firmware updates."); } # Output Check result info("Refer to KB516613 (https://www.dell.com/support/kbdoc/516613) for details.") if isResults(&INFO); printResults(); } # End checkKB516613 $CHECKS{'checkNodeFirmware'}{'Description'} = 'Checks node firmware for updates'; sub checkNodeFirmware { if (wasCalled()) { return } # Skip firmware checks if no firmware version to check against if (!$RPS{'IsiFw'} or $RPS{'IsiFw'} eq 'None') { return unsupported('No Node Firmware Package was selected'); } # Skip firmware checks if the cluster is virtual if (isVirtual()) { return unsupported('Unsupported on virtual platforms') } checkPatches(); checkPartitions(); checkKB521890(); checkKB516613(); checkKB519119(); checkKB489473(); loadRestrictedNFP(); our %NodeFirmware; our @NFP; my $MSG_BMC_PKG = 'Node firmware updates may not be accurate when a BMC firmware package is installed. Uninstall the BMC firmware package and re-run health checks.'; my $MSG_935_GEN4 = 'Node Firmware Package 9.3.5 selected as it contains the latest firmware versions for this cluster.'; my $MSG_935_ONEFS = 'Node Firmware Package 9.3.5 selected as the current and/or destination OneFS versions are less than OneFS 7.1.'; my $MSG_1035_ONEFS = 'Node Firmware Package 10.3.6 selected as the current and/or destination OneFS versions are less than OneFS 9.0'; my $MSG_11_GEN6 = 'Node Firmware Package 11.0 or later requires OneFS 9.0 or later and may contain additional node firmware updates.'; my $MSG_PS = 'There is a known issue that power supply firmware updates for S210 nodes may fail to update from 8.02 to 9.05 when running OneFS 7.1.1.0 to 7.1.1.4. For the power supply firmware to be updated, you must upgrade the OneFS version of the cluster to 7.1.1.5 or above.'; my $MSG_NVRAM1 = 'There is a known issue that NVRam LOx updates fail when running OneFS 7.1.1.0 to 7.1.1.8. For the NVRam to be updated, you must upgrade the OneFS version of the cluster to 7.1.1.9 or above.'; my $MSG_NVRAM2 = 'There is a known issue that NVRam LOx will not display or update its secondary component when running OneFS 7.1.1.0 to 7.1.1.5 or 7.2.0.0 to 7.2.0.3. For the NVRam secondary component to be displayed and updated, you must first upgrade the OneFS version of the cluster to a version not affected.'; my $MSG_QLOGIC = "If performing updates to QLogic firmware on OneFS $CLUSTER_INFO{'OneFS'}, a BXE patch must first be installed and all nodes with BXE network interfaces rebooted."; my $SPF_NEWER = 'The following nodes are on firmware versions that are newer than what is available in Isilon Node Firmware Package %s: %s'; my $SPF_VAR10 = 'To install the Node Firmware Package, a minimum of 500MB of free space is required in /var, the following nodes do not meet this requirement: %s'; my $SPF_VAR11 = 'To perform node firmware updates a minimum of 200MB of free space is required in /var, the following nodes do not meet this requirement: %s'; my $SPF_CMCSDR = 'The following nodes were identified as having a CMCSDR device reporting None: %s. Applying node firmware updates may be necessary to resolve CMCSDR devices reporting None.'; my $MSG_KB = 'Refer to KB 335177 (https://www.dell.com/support/kbdoc/335177) for details.'; ### # Selecting another node firmware package must occur before starting the # check so that the destination version is shown in the check output # Select 9.3.5 if running a OneFS < 7.1 and not upgrading to a OneFS >= 7.1 if (compareVersions($RPS{'IsiFw'}, '9.3.5') > 0 and isCurrentOneFS('<7.1') and !isDestinationOneFS('>=7.1')) { details($MSG_935_ONEFS); $RPS{'IsiFw'} = '9.3.5'; } # Select 9.3.5 if Gen4 nodes and no Gen5/6 or A100 nodes. if (compareVersions($RPS{'IsiFw'}, '9.3.5') > 0 and ($CLUSTER_INFO{'Gen4'} or $CLUSTER_INFO{'IQ'}) and !($CLUSTER_INFO{'Gen6'} or $CLUSTER_INFO{'Gen5'} or $CLUSTER_INFO{'A100'})) { $RPS{'IsiFw'} = '9.3.5'; details($MSG_935_GEN4); } # Select 10.3.6 if running OneFS < 9 and not upgrading to OneFS >= 9 if (compareVersions($RPS{'IsiFw'}, '11.0') >= 0 and isCurrentOneFS('<9') and !isDestinationOneFS('>=9')) { $RPS{'IsiFw'} = '10.3.6'; details($MSG_1035_ONEFS); # if ($CLUSTER_INFO{'Gen6'}) { details($MSG_11_GEN6) } } printCheck("Node Firmware ($RPS{'IsiFw'})"); # Results may not be accurate if a BMC firmware package is installed if (hasPatch('IsiFw.*BMC')) { fail($MSG_BMC_PKG) } gatherNodeFirmware(); gatherQLogicFirmware(); gatherLOxFirmware(); gatherNetworkInterfaces(); gatherPSRedundancy(); gatherCFFPSAssembly(); # Analyze information for issues my $power_supply_firmware_issue = 0; my $nvram_lox_issue = 0; my $nvram_lox_issue2 = 0; my @nvram_lox_issue3 = (); my @redundancy_lost = (); my $requires_bxe_patch = 0; foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'Firmware'}) { missing($lnn); next; } my $firmware_out_of_date = 0; my $firmware_is_newer = 0; DEVICE: foreach my $device (keys %{$NODE_INFO{$lnn}{'Firmware'}}) { if (!$device or !$NODE_INFO{$lnn}{'Firmware'}{$device}) { next } my $firmware = $NODE_INFO{$lnn}{'Firmware'}{$device}; debug("checkNodeFirmware: Node $lnn Device: $device Firmware: $firmware"); # Check for an issue where the LOx reports unknown information if ($device =~ /unknown:LOx/i and ! grep(/^lox/i, keys %{$NODE_INFO{$lnn}{'Firmware'}}) and isCurrentOneFS('<7.2.1')) { push(@nvram_lox_issue3, $lnn); debug("checkNodeFirmware: !!! Node $lnn is reporting unknown:LOx information"); } # Address when CFFPS1/2 is the device type but gathering the CFFPS assembly identifies differently. if ($NODE_INFO{$lnn}{$device}) { $device = $NODE_INFO{$lnn}{$device}; } # Check for firmware updates PACKAGE: foreach my $fwpkg (@NFP) { # Skip packages that are less than what was selected if (compareVersions($fwpkg, $RPS{'IsiFw'}) < 0) { next PACKAGE } # Device match if ($NodeFirmware{$fwpkg}{$device}) { # Firmware match if (grep {$firmware eq $_} @{$NodeFirmware{$fwpkg}{$device}}) { # Does not match the requested package if ($fwpkg ne $RPS{'IsiFw'}) { # Check if the firmware is also available in # the requested package or if it is newer # than available in the requested package my ($match) = grep {$firmware eq $_} @{$NodeFirmware{$RPS{'IsiFw'}}{$device}}; if (!$match) { $firmware_is_newer = 1 } } # Move to the next device next DEVICE; } } # The requested package has been reached, stop checking packages if ($fwpkg eq $RPS{'IsiFw'}) { last PACKAGE } } # Check for KB 489473 if ($device eq 'IsilonFPV1' and $firmware eq 'None') { push(@{$RPS{'Node Firmware'}{'489473'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by KB 489473"); next DEVICE; } # Updates for this device not available in the selected firmware package if (!$NodeFirmware{$RPS{'IsiFw'}}{$device}) { next DEVICE } # Firmware is out of date $firmware_out_of_date = 1; debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware out of date, valid recent firmware revisions: ".join(",", @{$NodeFirmware{$RPS{'IsiFw'}}{$device}})); push(@{$RPS{'Node Firmware'}{'Devices'}{$device}}, $lnn); # Check for ETA 184813 if ($device eq 'Lsi' and $firmware =~ /^[\d\.]+$/ and compareVersions($firmware, '10.00.06.00') <= 0) { push(@{$RPS{'ETA'}{'184813'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by ETA 184813"); } # Check for ETA 184406 if ($device eq 'Mellanox' and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^A100/ and $firmware =~ /^([\d\.]+)/ and compareVersions($1, '2.30.8020') < 0) { push(@{$RPS{'ETA'}{'184406'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by ETA 184406"); } # Check for KB466373/FCO F010517EE if ($device =~ '^CMC_' and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(S210|X210|X410|NL410|HD400)/ and compareVersions($firmware, '02.05') < 0) { push(@{$RPS{'FCO'}{'F010517EE'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F010517EE"); } # Check for KB513206/FCO F121917EE if ($device eq 'EPpost_warthog' and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^(H500|H600|F800)/ and compareVersions($firmware, '20.81') < 0) { push(@{$RPS{'FCO'}{'F121917EE'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F121917EE"); } # Check for KB515633/FCO F011018EE if ($device eq 'EPsas_infinity' and $NODE_INFO{$lnn}{'Product'} and $NODE_INFO{$lnn}{'Product'} =~ /^F800/ and compareVersions($firmware, '0001.0004.0000.11181') < 0) { push(@{$RPS{'FCO'}{'F011018EE'}{'Nodes'}}, $lnn); debug("checkNodeFirmware: !!! Node $lnn Device $device Firmware $firmware affected by FCO F011018EE"); } # Check for known issue with S210 Power Supply firmware updates on OneFS versions prior to 7.1.1.5. if ($device =~ /Octane/ and $firmware =~ /8\.02/ and isCurrentOneFS('<7.1.1.5')) { $power_supply_firmware_issue = 1; debug("checkNodeFirmware: !!! Known issue with S210 nodes and power supply firmware updates on OneFS versions 7.1.1.[01234]"); } if ($device =~ /LOx/ and isCurrentOneFS('<7.1.1.9')) { $nvram_lox_issue = 1; debug("checkNodeFirmware: !!! Known issue with LOx and OneFS 7.1.1.[012345678] branch: https://www.dell.com/support/kbdoc/465882"); } if ($device =~ /LOx/ and !($firmware =~ /\+/)) { $nvram_lox_issue2 = 1; debug("checkNodeFirmware: !!! Known issue with LOx and OneFS 7.2.0.[0123] and OneFS 7.1.1.[012345] where secondary firmware does not show or get updated in these versions"); } # A flag if QLogic firmware updates are needed if ($device =~ /QLogic-NX2/ and compareVersions($firmware, '7.13.0') < 0) { $RPS{'Node Firmware'}{'QLogic'} = 1; } # Flag CMCSDR reporting None and recommend firmware updates to fix it if ($device =~ /^CMCSDR/ and $firmware eq 'None') { debug("checkNodeFirmware: !!! Known issue with CMCSDR reporting None"); push(@{$RPS{'Node Firmware'}{'CMCSDR'}}, $lnn); } } if ($firmware_out_of_date) { push(@{$RPS{'Node Firmware'}{'Nodes'}}, $lnn); } elsif ($firmware_is_newer) { push(@{$RPS{'Node Firmware'}{'Newer'}}, $lnn); } # Check for Gen6 nodes having fully redundant power supplies if ($NODE_INFO{$lnn}{'Power Supply Redundancy'} and $NODE_INFO{$lnn}{'Power Supply Redundancy'} ne 'Fully Redundant') { push(@redundancy_lost, $lnn) } } # Flag issue where Gen6 nodes do not have fully redundant power supplies which will cause node firmware updates to error. if (@redundancy_lost) { fail('The following nodes are not reporting "Fully Redundant" power supplies: '.compressRange(@redundancy_lost)); } # NVRam LOx known issue should be displayed regardless of updates being available if (@nvram_lox_issue3) { info('Unable to determine if NVRam LOx firmware updates are needed due to it reporting .'); info('If NVRam LOx updates are needed, you must first upgrade the OneFS version of the cluster to 7.1.1.9 or above.') if isCurrentOneFS('<=7.1.1.8'); } # Node Firmware updates available if ($RPS{'Node Firmware'}{'Nodes'}) { # Show nodes requiring updates info(scalar(@{$RPS{'Node Firmware'}{'Nodes'}}).' nodes have out of date node firmware: '.compressRange(@{$RPS{'Node Firmware'}{'Nodes'}})); debug('checkNodeFirmware: !!! Devices with out of date firmware: '.join(', ', keys %{$RPS{'Node Firmware'}{'Devices'}})); # Display ETAs and known issues if ($RPS{'ETA'}{'184406'}{'Nodes'}) { $RPS{'ETA'}{'184406'}{'Severity'} = 'INFO'; $RPS{'ETA'}{'184406'}{'Message'} = scalar(@{$RPS{'ETA'}{'184406'}{'Nodes'}}).' nodes are running firmware versions affected by ETA 184406: '.compressRange(@{$RPS{'ETA'}{'184406'}{'Nodes'}}); warning($RPS{'ETA'}{'184406'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'ETA'}{'184406'}{'Message'}"); } if ($RPS{'ETA'}{'184813'}{'Nodes'}) { $RPS{'ETA'}{'184813'}{'Severity'} = 'INFO'; $RPS{'ETA'}{'184813'}{'Message'} = scalar(@{$RPS{'ETA'}{'184813'}{'Nodes'}}).' nodes are running firmware versions affected by ETA 184813: '.compressRange(@{$RPS{'ETA'}{'184813'}{'Nodes'}}); warning($RPS{'ETA'}{'184813'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'ETA'}{'184813'}{'Message'}"); } if ($RPS{'FCO'}{'F010517EE'}{'Nodes'}) { $RPS{'FCO'}{'F010517EE'}{'Severity'} = 'INFO'; $RPS{'FCO'}{'F010517EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F010517EE'}{'Nodes'}}).' nodes are running firmware versions affected by FCO F010517EE: '.compressRange(@{$RPS{'FCO'}{'F010517EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F010517EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F010517EE'}{'Message'}"); } if ($RPS{'FCO'}{'F121917EE'}{'Nodes'}) { $RPS{'FCO'}{'F121917EE'}{'Severity'} = 'INFO'; $RPS{'FCO'}{'F121917EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F121917EE'}{'Nodes'}}).' nodes are running firmware versions affected by FCO F121917EE: '.compressRange(@{$RPS{'FCO'}{'F121917EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F121917EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F121917EE'}{'Message'}"); my $msg = 'Certain voltage threshold monitoring values were incorrectly programmed to be too stringent for H500, H600, and F800 nodes. This could result in unnecessary preventative action by the system, such as powering down. This issue is resolved in Node Firmware Package 10.1.1 and later, but before you perform a node firmware update on F800, H600, or H500 nodes, you must either install patch-211361 or run the update_dpol_setpoint.py script based on the OneFS version of the cluster.'; push(@{$RPS{'Node Firmware'}{'Priority'}}, $msg); critical($msg); if (not hasPatch('patch-211361') and isCurrentOneFS(qr/^8\.1\.0\.[01]$/)) { $RPS{'Node Firmware'}{'Patch'}{'patch-211361'}{'Impacts'} = ''; } } if ($RPS{'FCO'}{'F011018EE'}{'Nodes'}) { $RPS{'FCO'}{'F011018EE'}{'Severity'} = 'INFO'; $RPS{'FCO'}{'F011018EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F011018EE'}{'Nodes'}}).' nodes are running firmware versions affected by FCO F011018EE: '.compressRange(@{$RPS{'FCO'}{'F011018EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F011018EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F011018EE'}{'Message'}"); } if ($RPS{'FCO'}{'F082318EE'}{'Nodes'}) { $RPS{'FCO'}{'F082318EE'}{'Severity'} = 'INFO'; $RPS{'FCO'}{'F082318EE'}{'Message'} = scalar(@{$RPS{'FCO'}{'F082318EE'}{'Nodes'}}).' nodes are running node firmware versions affected by FCO F082318EE: '.compressRange(@{$RPS{'FCO'}{'F082318EE'}{'Nodes'}}); warning($RPS{'FCO'}{'F082318EE'}{'Message'}); push(@{$RPS{'Node Firmware'}{'Priority'}}, "Node firmware updates are strongly recommended because $RPS{'FCO'}{'F082318EE'}{'Message'}"); } if ($power_supply_firmware_issue) { info($MSG_PS) } if ($nvram_lox_issue) { info($MSG_NVRAM1) } if ($nvram_lox_issue2) { info($MSG_NVRAM2) } if ($RPS{'Node Firmware'}{'QLogic'} and isCurrentOneFS(qr/^7\.1\.1\.([0123456789]|10|11)$|^7\.2\.0\.[012345]$|^7\.2\.1\.[012345]$|^8\.0\.0\.[01234]/) and not hasPatch('patch-(188239|189133|175451)')) { $requires_bxe_patch = 1; critical($MSG_QLOGIC); } if ($RPS{'Node Firmware'}{'var10'} and !hasPatch('^IsiFw') and compareVersions($RPS{'IsiFw'}, '10.0') >= 0 and compareVersions($RPS{'IsiFw'}, '11.0') < 0 and isCurrentOneFS('<9.0')) { fail(sprintf($SPF_VAR10, compressRange(@{$RPS{'Node Firmware'}{'var10'}}))); } if ($RPS{'Node Firmware'}{'var11'} and !hasPatch('^IsiFw') and compareVersions($RPS{'IsiFw'}, '11.0') >= 0) { fail(sprintf($SPF_VAR11, compressRange(@{$RPS{'Node Firmware'}{'var11'}}))); } if ($RPS{'Node Firmware'}{'CMCSDR'}) { critical(sprintf($SPF_CMCSDR, compressRange(@{$RPS{'Node Firmware'}{'CMCSDR'}}))); } if ($RPS{'Node Firmware'}{'Newer'}) { info(sprintf($SPF_NEWER, $RPS{'IsiFw'}, compressRange(@{$RPS{'Node Firmware'}{'Newer'}}))); } # Add the package to required files push(@{$RPS{'Node Firmware'}{'Files'}}, "Isilon Node Firmware Package $RPS{'IsiFw'}"); # Add the target version for the node firmware updates $RPS{'Node Firmware'}{'Target'} = "Node Firmware $RPS{'IsiFw'}"; # Adjust order based on current OneFS version and known issues $RPS{'Node Firmware'}{'Position'} = 1; $RPS{'Node Firmware'}{'Position'} = 2 if isCurrentOneFS('<6.5.2.85'); $RPS{'Node Firmware'}{'Position'} = 2 if isCurrentOneFS('<7.1') and compareVersions($RPS{'IsiFw'}, '10.0.1') >= 0; $RPS{'Node Firmware'}{'Position'} = 2 if ($power_supply_firmware_issue and isDestinationOneFS('>=7.1.1.5')); $RPS{'Node Firmware'}{'Position'} = 2 if ($nvram_lox_issue and isDestinationOneFS('>=7.1.1.9')); $RPS{'Node Firmware'}{'Position'} = 2 if ($nvram_lox_issue2 and isDestinationOneFS() and !isDestinationOneFS(qr/^7\.1\.1\.[012345]|^7\.2\.0\.[0123]/)); $RPS{'Node Firmware'}{'Position'} = 2 if (@nvram_lox_issue3 and isDestinationOneFS('>=7.1.1.9')); $RPS{'Node Firmware'}{'Position'} = 2 if $requires_bxe_patch; $RPS{'Node Firmware'}{'Position'} = 2 if !isDestinationOneFS(); $RPS{'Node Firmware'}{'Position'} = 2 if $RPS{'KB521890'}; $RPS{'Node Firmware'}{'Position'} = 2 if $RPS{'KB519890'}; $RPS{'Node Firmware'}{'Position'} = 2 if !$RPS{'KB516613'} and !$RPS{'Node Firmware'}{'CMCSDR'} and isDestinationOneFS('>=8.1.0.4') and isCurrentOneFS('<8.1.0.4'); $RPS{'Node Firmware'}{'Position'} = 2 if compareVersions($RPS{'IsiFw'}, '11.0') >= 0 and isCurrentOneFS('<9.0') and isDestinationOneFS('>=9.0'); # Parallel method requested and available in destination OneFS version if (exists $RPS{'Node Firmware'}{'Method'} and $RPS{'Node Firmware'}{'Method'} eq 'parallel' and isCurrentOneFS('<8.2') and isDestinationOneFS('>=8.2')) { $RPS{'Node Firmware'}{'Position'} = 2; } # Remove patch-211361 install if node firmware is after OneFS if ($RPS{'Node Firmware'}{'Position'} == 2 and $RPS{'Node Firmware'}{'Patch'} and $RPS{'Node Firmware'}{'Patch'}{'patch-211361'} and isDestinationOneFS('>=8.1.0.2')) { delete $RPS{'Node Firmware'}{'Patch'}; } if ($RPS{'F011018EE'}) { push(@{$RPS{'Node Firmware'}{'Additional Comments'}}, 'Start IntegrityScan job after node firmware updates'); } # Remove node firmware updates from upgrade plan if requested only # priority updates and there are no priority updates needed. if ($RPS{'Priority NF Only'} and !$RPS{'Node Firmware'}{'Priority'}) { delete $RPS{'Node Firmware'}; } } # Add Node Firmware Package to required files for use post OneFS 9.1 if (!$RPS{'Node Firmware'}{'Target'} and isOneFSUpgrade() and isDestinationOneFS('>=9.1')) { addRequiredFiles("Isilon Node Firmware Package $RPS{'IsiFw'}"); } # Flag if parallel was requested but not available my $MSG_NFP_PARALLEL = 'Parallel method for node firmware updates was'. ' requested, but is not available prior to OneFS 8.2 release, rolling'. ' method will be used'; if (exists $RPS{'Node Firmware'}{'Position'} and exists $RPS{'Node Firmware'}{'Method'} and $RPS{'Node Firmware'}{'Method'} eq 'parallel' and (($RPS{'Node Firmware'}{'Position'} == 1 and isCurrentOneFS('<8.2')) or ($RPS{'Node Firmware'}{'Position'} == 2 and (isDestinationOneFS('<8.2') or (!isDestinationOneFS() and isCurrentOneFS('<8.2')))))) { warning($MSG_NFP_PARALLEL); $RPS{'Node Firmware'}{'Method'} = 'rolling'; } # Output Check result if (isResults(&INFO)) { info($MSG_KB) } printResults(); } # End checkNodeFirmware ### # gatherCFFPSAssembly: Gathers the Product Part/Model Number for the # CFFPS power supplies on Generation 5 nodes. sub gatherCFFPSAssembly { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen5'}) { return } my $SPF_CMD_LOGS = q[2>&1 grep -H '071-' */cff_ps%s_info]; my $nodelist = compressRange(@{$CLUSTER_INFO{'Gen5'}}); my $cmd = '/usr/bin/isi_hwtools/isi_ipmicmc -F -V -p D -a cmc -i %s'; my $SPF_CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array -n $nodelist '${SUDO} $cmd | grep "071-"']; # Gather information my $gather1 = (); my $gather2 = (); if ($RPS{'Live'}) { $gather1 = gatherCluster(sprintf($SPF_CMD_LIVE, 1)); $gather2 = gatherCluster(sprintf($SPF_CMD_LIVE, 2)); } else { $gather1 = gatherCluster(sprintf($SPF_CMD_LOGS, 1)); $gather2 = gatherCluster(sprintf($SPF_CMD_LOGS, 2)); } # Analyze information for issues. foreach my $lnn (@{$CLUSTER_INFO{'Gen5'}}) { if (!$gather1->{$lnn} or !$gather2->{$lnn}) { missing($lnn); next; } foreach (@{$gather1->{$lnn}}) { if (!/(071-.*)$/) { next } $NODE_INFO{$lnn}{'CFFPS1'} = join('_', 'CFFPS1', getCFFPSName($_)); debug("NODE_INFO->${lnn}->CFFPS1=$NODE_INFO{$lnn}{'CFFPS1'}"); } foreach (@{$gather2->{$lnn}}) { if (!/(071-.*)$/) { next } $NODE_INFO{$lnn}{'CFFPS2'} = join('_', 'CFFPS2', getCFFPSName($_)); debug("NODE_INFO->${lnn}->CFFPS2=$NODE_INFO{$lnn}{'CFFPS2'}"); } } } # End gatherCFFPSAssembly ### # gatherFirmwareStatusJSON: Gathers node firmware status from /var/ifs/upgrade/firmware_status.json sub gatherFirmwareStatusJSON { if (wasCalled()) { return } if (isCurrentOneFS('<8.0')) { return } my $CMD_NODE = 'cat /var/ifs/upgrade/firmware_status.json;echo'; my $CMD_LOGS = 'grep -H . */upgrade_local.tar/var/ifs/upgrade/firmware_status.json'; # Gathers from /var/ifs/upgrade/firmware_status.json my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array '$CMD_NODE']); } else { $gather = gatherCluster(qq[2>&1 $CMD_LOGS]); } foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach my $line (@{$gather->{$lnn}}) { my @devices = split(/\}\s*,\s*\{/, $line); foreach my $device_info (@devices) { my ($device, $firmware) = ""x2; if ($device_info =~ /"device"[^"]+"([^"]+)"/) { $device = $1 } if ($device_info =~ /"version"[^"]+"([^"]+)"/) { $firmware = $1 } if (!$device or !$firmware) { next } $NODE_INFO{$lnn}{'Firmware'}{$device} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${device}=$firmware"); } } } } # gatherFirmwareStatusJSON ### # gatherFirmwareStatus: Gathers node firmware sub gatherFirmwareStatus { if (wasCalled()) { return } # Gathers from isi firmware status, even in newer OneFS versions! If # information is found here it replaces previously found info. my $gather = (); if ($RPS{'Live'}) { if (isCurrentOneFS('>=8')) { $gather = (); } else { $gather = gatherLocal(qq[2>&1 ${SUDO} isi firmware status]); } } else { $gather = gatherLocal(q[2>&1 cat firmware local/firmware]); } foreach my $line (@$gather) { # Ignore informational lines if ($line =~ /^\s*(Device|NOTE|WARNING|-)/) { next } # Split by whitespace, removing groups of just whitespace my @columns = grep { /\S/ } split(/\s+/, $line); if (!@columns) { next } # Remove the column if it is a * indicating out of date firmware if ($columns[0] eq '*') { shift @columns } # Must be at least four columns if (!@columns or scalar @columns < 4) { next } # The nodes column must only consist of nodes if ($columns[-1] !~ /^[0-9,-]+$/) { next } # Assign out firmware information my $device = $columns[0]; my $type = $columns[1]; my $firmware = $columns[2]; my @nodes = expandRange($columns[-1]); foreach my $lnn (@nodes) { # Make sure lnn is a valid node if (!$NODE_INFO{$lnn}) { next } $NODE_INFO{$lnn}{'Firmware'}{$device} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${device}=$firmware"); if ($type eq 'CMC' and not $NODE_INFO{$lnn}{'Firmware'}{'CMC'}) { $NODE_INFO{$lnn}{'Firmware'}{'CMC'} = $firmware; debug("NODE_INFO->${lnn}->Firmware->CMC=$firmware"); } } } } # gatherFirmwareStatus ### # gatherFirmwareVersions: Gathers node firmware sub gatherFirmwareVersions { if (wasCalled()) { return } # Gathers from /etc/ifs/firmware_versions, this is information that is # updated during node reboots and /may/ be inaccurate! my $gather = (); if ($RPS{'Live'}) { my $cmd = 'cat /etc/ifs/firmware_versions'; $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array '$cmd']); } else { $gather = gatherCluster(q[2>&1 grep -H . */etcifs.tar/ifs/firmware_versions]); } foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach my $line (@{$gather->{$lnn}}) { # Ignore informational lines if ($line =~ /^\s*(Device|NOTE|WARNING|-|Writing out firmware status JSON)/) { next } # Split by whitespace, removing groups of just whitespace my @columns = grep { /\S/ } split(/\s+/, $line); if (!@columns) { next } # Remove the column if it is a * indicating out of date firmware if ($columns[0] eq '*') { shift @columns } # Must be at least four columns if (!@columns or scalar @columns < 4) { next } # Assign out firmware information my $device = $columns[0]; my $type = $columns[1]; my $firmware = $columns[2]; $NODE_INFO{$lnn}{'Firmware'}{$device} = $firmware; debug("NODE_INFO->${lnn}->Firmware->${device}=$firmware"); } } } # gatherFirmwareVersions ### # gatherLOxFirmware: Gathers ( or attempts to gather ) LOx NVRAM firmware revision information. # Currently only works on a live cluster, only designed to address an issue in the 7.1.1 branch. sub gatherLOxFirmware { return if wasCalled(); return unless $RPS{'Live'}; return unless isCurrentOneFS(qr[^7\.1\.1\.]); gatherProductInfo(); return unless $CLUSTER_INFO{'Gen5'}; # Gather information from the cluster my $gather = (); $gather = gatherCluster(qq[ 2>&1 ${SUDO} isi_for_array -n ]. compressRange(@{$CLUSTER_INFO{'Gen5'}}). qq[ '/sbin/nvmecontrol identify lnv0; ]. qq[ /sbin/nvmecontrol devinfo lnv0' ]); # Look for firmware versions and assign based on found information foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; # If LOx was already gathered, skip to the next node. next if $NODE_INFO{$lnn}{'Firmware'}{'LOx'} or $NODE_INFO{$lnn}{'Firmware'}{'LOx2'} or $NODE_INFO{$lnn}{'Firmware'}{'LOx2-MLC-YD'}; # Reset variables my $primary = ""; my $secondary = ""; my $model = ""; # Gather information for the node foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /Firmware Version:\s+(\S+)/) { $primary = $1; debug("gatherLOxFirmware: Primary Firmware Version found: $primary"); } elsif ($line =~ /Secondary Boot Loader Rev:\s+(\S+)/) { $secondary = $1; debug("gatherLOxFirmware: Secondary Firmware Version found: $secondary"); } elsif ($line =~ /Model Number:\s+(\S+)/) { $model = $1; debug("gatherLOxFirmware: Model found: $model"); } } # A minimum of the model and the primary firmware version is required next unless $model and $primary; # Assign device by the model my $device = ""; if ($model =~ /303-409-000A/) { $device = "LOx2"; } elsif ($model =~ /303-409-001[AB]/) { $device = "LOx2-MLC-YD"; } else { $device = "LOx"; } debug("gatherLOxFirmware: Model identified as device: $device"); # Assign firmware information $NODE_INFO{$lnn}{'Firmware'}{$device} = "$primary".($secondary?"+$secondary":""); debug("NODE_INFO->${lnn}->Firmware->$device=$NODE_INFO{$lnn}{'Firmware'}{$device}"); # Undefine any unknown LOx device information found my @unknown_devices = grep(/unknown:LOx/, keys %{$NODE_INFO{$lnn}{'Firmware'}}); foreach my $unknown_device (@unknown_devices) { delete $NODE_INFO{$lnn}{'Firmware'}{$unknown_device}; } } } # End gatherLOxFirmware ### # gatherNodeFirmware: Gathers node firmware sub gatherNodeFirmware { if (wasCalled()) { return } gatherFirmwareVersions(); gatherFirmwareStatusJSON(); gatherFirmwareStatus(); } # gatherNodeFirmware sub gatherPSRedundancy { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'Gen6'}) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array -n ]. compressRange(@{$CLUSTER_INFO{'Gen6'}}). qq[ '${SUDO} isi_hwmon -r' ]; my $CMD_LOGS8 = q[2>&1 grep -H . */hwmon_psu]; my $CMD_LOGS9 = q[2>&1 grep -H 'Power Supply Redundancy' */isi_hwmon]; my $SPF = '%5s %s'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } elsif (isCurrentOneFS('>=9')) { $gather = gatherCluster($CMD_LOGS9); } else { $gather = gatherCluster($CMD_LOGS8); } # Save verboseHeader(sprintf($SPF, 'LNN', 'Power Supply Redundancy')); foreach my $lnn (@{$CLUSTER_INFO{'Gen6'}}) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { if ($line !~ /Power Supply Redundancy:\s+(\S.*)$/) { next } verbose(sprintf($SPF, $lnn, $1)); $NODE_INFO{$lnn}{'Power Supply Redundancy'} = $1; debug("NODE_INFO=>${lnn}=>Power Supply Redundancy=$1"); } } } # End gatherPSRedundancy ### # gatherQLogicFirmware: Gathers ( or attempts to gather ) QLogic BXE firmware revision information. # This is will work 100% of the time live on a cluster but only works on logsets when the log gather # was performed with an FRU/CTO package of 201506 or newer.. sub gatherQLogicFirmware { return if wasCalled(); gatherProductInfo(); return unless $CLUSTER_INFO{'Gen5'}; # Gather information from the cluster my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[ 2>&1 ${SUDO} isi_for_array -n ]. compressRange(@{$CLUSTER_INFO{'Gen5'}}). qq[ sysctl dev.bxe.0.bc_version ]); } else { $gather = gatherCluster(qq[ 2>&1 grep -H . */asbuilt.xml ]); } # Look for firmware versions and assign based on found information foreach my $lnn (@LNNS) { # Skip if gather has nothing next unless $gather->{$lnn}; # Skip if already gathered next if $NODE_INFO{$lnn}{'Firmware'}{'QLogic-NX2'}; # Parse output foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /dev\.bxe\.0\.bc_version:\s+(\S+)$/ or ($line =~ /BCM57810/ and $line =~ /firmware=\"(.*?)\"/); $NODE_INFO{$lnn}{'Firmware'}{'QLogic-NX2'} = $1; debug("NODE_INFO->${lnn}->Firmware->QLogic-NX2=$1"); } } } # End gatherQLogicFirmware ### # getCFFPSName: Given an assembly number, return the corresponding name sub getCFFPSName { if (!@_) { return } ($_) = @_; if (/071-000-555(-01)?/ or /071-000-597(-00)?/ or /071-000-022(-00)?/) { return 'Octane'; } elsif (/071-000-578(-01)?/) { return 'Blastoff'; } elsif (/071-000-580(-0[45])?/) { return 'Ramjet'; } elsif (/071-000-012(-03)?/ or /071-000-608(-0[01])?/) { return 'Optimus_Flex'; } elsif (/071-000-036(-02)?/ or /071-000-609(-01)?/) { return 'Optimus_Acbel'; } } # End getCFFPSName # Temporary sub-routine to load into the NodeFirmware and NFP globals restricted Node Firmware Packages that are newer than currently available packages. sub loadRestrictedNFP { our %NodeFirmware; our @NFP; $NodeFirmware{'11.3'} = { 'BIOS' => ['2.8.2'], 'BMCSDR_A100' => ['01.07'], 'Backplane 1' => ['2.46', '4.35'], 'Backplane 2' => ['4.35'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['02.20'], 'DEbios_banshee' => ['37.41'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.51'], 'DEccdes' => ['1.10.4+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.41'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.15'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0004.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11192'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.02'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'Dell HBA330 Mini' => ['16.17.01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['02.20'], 'EPbios_warthog' => ['32.51'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.51'], 'EPccdes' => ['1.10.4+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.51'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.14'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0004.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11192'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['08.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'Integrated Remote Access Controller' => ['4.40.10.00'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'MLKadpt_skylanders' => ['02.80'], 'MLKbem_mongoose' => ['02.20'], 'MLKbios_gryphon' => ['10.57'], 'MLKccdes' => ['1.10.4+21.00'], 'MLKcdes' => ['2.41.1+0.18.0+21.00'], 'MLKcec_region0' => ['02.00'], 'MLKcec_region1' => ['32.02'], 'MLKcec_region3' => ['00.03'], 'MLKcmd_100dc12pin' => ['02.02.72.03'], 'MLKcmd_64pin' => ['02.05.102.02'], 'MLKcmd_rev_table' => ['20.66'], 'MLKeth_infinitymlk' => ['02.64'], 'MLKfp_fpinffw' => ['02.0c'], 'MLKfp_fpioc' => ['00.0b'], 'MLKfullbios_gryphon_post' => ['10.57'], 'MLKfw_map_table' => ['40.49'], 'MLKgem_skylanders' => ['45.60'], 'MLKpost_gryphon' => ['10.32'], 'MLKps0_gen2_artesyn_optimus' => ['02.14'], 'MLKps0_gen2_artesyn_ramjet' => ['02.14'], 'MLKps1_gen2_artesyn_optimus' => ['02.14'], 'MLKps1_gen2_artesyn_ramjet' => ['02.14'], 'MLKsas_catamaran' => ['0015.0004.0000.0000'], 'MLKsas_infinitymlk' => ['0001.0004.0000.11192'], 'MLKserdes_infinitymlk' => ['00.40'], 'MLKspime_gryphon' => ['04.00'], 'MLKssp_infinitymlk' => ['01.40'], 'MLKuefi_fw_volume' => ['40.86'], 'MLKvrd0_infinitymlk_infineon' => ['01.01'], 'MLKvrd1_infinitymlk_infineon' => ['01.01'], 'MLKvrd2_infinitymlk_infineon' => ['01.01'], 'MLKvrd3_infinitymlk_infineon' => ['01.02'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-CX4' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4-BE' => ['14.27.6008+DEL2420110034'], 'Mellanox-CX4R' => ['14.27.6008+DEL2810000034'], 'Mellanox-CX5-BE' => ['16.27.6106+DEL0000000004'], 'Mellanox-CX5-FE0' => ['16.27.6106+DEL0000000004'], 'Mellanox-CX6-BE' => ['22.28.4512+DEL0000000027'], 'Mellanox-CX6-FE0' => ['22.28.4512+DEL0000000027'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['16.28.1002+EMC0000000017'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'Port 0: Emulex LightPulse LPe31002-M6-D 2-Port 16Gb Fibre - FC' => ['03.04.24'], 'Port 1: Emulex LightPulse LPe31002-M6-D 2-Port 16Gb Fibre - FC' => ['03.04.24'], 'QLogic-NX2' => ['7.13.0'], }; $NodeFirmware{'10.3.7'} = { 'BMCSDR_A100' => ['01.07'], 'CAR_Colossus' => ['02.07'], 'CAR_Ermine' => ['02.07'], 'CFFPS1' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS1_Blastoff' => ['03.09'], 'CFFPS1_Octane' => ['09.0b'], 'CFFPS1_Octane_Acbel' => ['0a.01'], 'CFFPS1_Optimus' => ['04.14', '02.02'], 'CFFPS1_Optimus_Acbel' => ['02.02'], 'CFFPS1_Optimus_Flex' => ['04.14'], 'CFFPS1_Ramjet' => ['06.06'], 'CFFPS1_Ramjet_Acbel' => ['01.03'], 'CFFPS2' => ['0a.01', '01.03', '04.14', '03.09', '06.06', '09.0b', '02.02'], 'CFFPS2_Blastoff' => ['03.09'], 'CFFPS2_Octane' => ['09.0b'], 'CFFPS2_Octane_Acbel' => ['0a.01'], 'CFFPS2_Optimus' => ['04.14', '02.02'], 'CFFPS2_Optimus_Acbel' => ['02.02'], 'CFFPS2_Optimus_Flex' => ['04.14'], 'CFFPS2_Ramjet' => ['06.06'], 'CFFPS2_Ramjet_Acbel' => ['01.03'], 'CMCSDR_Colossus' => ['00.08'], 'CMCSDR_Ermine' => ['00.03'], 'CMCSDR_Honeybadger' => ['00.0B'], 'CMCSDR_Hotfoot' => ['00.0B'], 'CMCSDR_Yeti' => ['00.06'], 'CMC_Colossus' => ['02.07'], 'CMC_Ermine' => ['02.07'], 'CMC_HFHB' => ['02.07'], 'CMC_Yeti' => ['02.07'], 'CMD_Colossus' => ['02.0b+49+01'], 'CMD_Ermine' => ['02.0d+4c+01'], 'DEadpt_moons' => ['01.80'], 'DEbcc_infinity' => ['02.20'], 'DEbios_banshee' => ['37.41'], 'DEbmc_bblock_moons' => ['01.40'], 'DEbmc_main' => ['24.51'], 'DEccdes' => ['1.10.4+21.00'], 'DEcdes' => ['2.38.5+0.18.0+21.00'], 'DEcmd_100d815pin' => ['03.08.103.01'], 'DEcmd_64pin' => ['02.05.102.02'], 'DEcmd_cat_64pin' => ['02.05.102.02'], 'DEcmd_mam_64pin' => ['02.05.102.02'], 'DEcmd_rev_table' => ['20.45'], 'DEeth_infinity' => ['02.52'], 'DEeth_sageville_infinity' => ['0128.0000.0005.0195'], 'DEfp_fpinffw' => ['02.0c'], 'DEfp_fpioc' => ['00.0b'], 'DEfullbios_banshee_post' => ['37.41'], 'DEfw_map_table' => ['12.17'], 'DEpost_banshee' => ['28.15'], 'DEps0_gen2_acbel_hv_zeus' => ['04.12'], 'DEps0_gen2_artesyn' => ['02.14'], 'DEps1_gen2_acbel_hv_zeus' => ['04.12'], 'DEps1_gen2_artesyn' => ['02.14'], 'DEsas_catamaran' => ['0015.0004.0000.0000'], 'DEsas_infinity' => ['0001.0004.0000.11192'], 'DEserdes_banshee' => ['08.00'], 'DEspime_banshee' => ['02.02'], 'DEssp_infinity' => ['02.83'], 'DEuefi_fw_volume_infinity' => ['03.43'], 'DEvrd0_banshee_stmicro' => ['02.01'], 'DEvrd1_banshee_stmicro' => ['01.00'], 'EPadpt_moons' => ['01.80'], 'EPbcc_infinity' => ['02.20'], 'EPbios_warthog' => ['32.51'], 'EPbmc_bblock_moons' => ['01.40'], 'EPbmc_main' => ['24.51'], 'EPccdes' => ['1.10.4+21.00'], 'EPcdes' => ['2.38.5+0.18.0+21.00'], 'EPcmd_100d815pin' => ['03.08.103.01'], 'EPcmd_64pin' => ['02.05.102.02'], 'EPcmd_cat_64pin' => ['02.05.102.02'], 'EPcmd_mam_64pin' => ['02.05.102.02'], 'EPcmd_rev_table' => ['20.45'], 'EPeth_infinity' => ['02.52'], 'EPfp_fpinffw' => ['02.0c'], 'EPfp_fpioc' => ['00.0b'], 'EPfullbios_warthog_post' => ['32.51'], 'EPfw_map_table' => ['12.17'], 'EPpost_warthog' => ['22.14'], 'EPps0_gen2_acbel_hv_zeus' => ['04.12'], 'EPps0_gen2_artesyn' => ['02.14'], 'EPps1_gen2_acbel_hv_zeus' => ['04.12'], 'EPps1_gen2_artesyn' => ['02.14'], 'EPsas_catamaran' => ['0015.0004.0000.0000'], 'EPsas_infinity' => ['0001.0004.0000.11192'], 'EPserdes_warthog' => ['08.00'], 'EPspime_warthog' => ['08.00'], 'EPssp_infinity' => ['02.83'], 'EPuefi_fw_volume_infinity' => ['03.43'], 'EPvrd0_warthog_stmicro' => ['01.01'], 'EPvrd1_warthog_stmicro' => ['01.01'], 'EPvrd2_warthog_stmicro' => ['01.01'], 'Emulex LightPulse FC Adapter' => ['11.4.204.21'], 'IsilonFPV1' => ['UI.01.36'], 'IsilonIB' => ['4.8.930+205-0002-05_A'], 'LOx' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2' => ['rp180d01', 'rp180d01+rp180d01'], 'LOx2-MLC-YD' => ['rp180d01', 'rp180d01+rp180d01'], 'Lsi' => ['13.00.66.00', '20.00.04.00'], 'Lsi-3008 SAS Controller' => ['15.00.00.00+17.00.03.00'], 'Lsi-35x36 SAS Expander' => ['1.10.0+21.00'], 'LsiExp0' => ['0910+0210'], 'LsiExp1' => ['0910+0210'], 'Mellanox' => ['4.8.930+MT_02F0140002', '2.40.5030+MT_1090110018', '2.40.5030+ISL1090110018'], 'Mellanox-EN' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023'], 'Mellanox-EN0' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-EN1' => ['2.40.5030+MT_1090111023', '2.40.5030+EMC1090111023', '2.40.5030+EMC0000000002'], 'Mellanox-FPGA' => ['0x1002+0x1002', '0x100a+0x100a'], 'Mellanox0' => ['2.40.5030+EMC0000000004'], 'Mellanox1' => ['2.40.5030+EMC0000000004'], 'Mellanox5-EN0' => ['14.25.1020+DEL2420110034', '16.23.4010+MT_0000000158', '16.28.1002+EMC0000000017'], 'PMC8056' => ['2.38.5+0.18.0+21.00'], 'QLogic-NX2' => ['7.13.0'], 'XL710' => ['5.04+80002505'], }; # Add the newer packages to the front of the NFP list unshift(@NFP, '11.3','10.3.7'); } ### # checkAccessZones: A very dreaded failure, flags if multiple access # zones are configured for upgrades across 7.1.1. $CHECKS{'checkAccessZones'}{'Description'} = "Checks if multiple access zones are configured for upgrades across 7.1.1. Checks for nested or overlapping SMB shares. Adds warnings when any non-System access zone pools are found, only applies for upgrades across OneFS 7.1.1"; sub checkAccessZones { if (wasCalled()) { return } # Skip if OneFS currently less than 7.0 if (isCurrentOneFS('<7.0')) { debug("checkAccessZones: Current OneFS version less than 7.0, skipping."); return; } printCheck("Access Zones"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi auth local list"); } elsif (-e "local/isi_auth_local_list") { $gather = gatherLocal("2>&1 cat local/isi_auth_local_list"); } elsif (-e "local/isi_auth") { $gather = gatherLocal("2>&1 cat local/isi_auth"); } # Analyze information for issues. my $parse_total = $RPS{'Live'}; foreach my $line (@$gather) { debug("checkAccessZones: $line"); $parse_total = 1 if $line =~ /auth local list$/; if ($parse_total and $line =~ /^\s*Total:\s*(\d*)$/) { $CLUSTER_INFO{'Access Zones'} = $1; debug("checkAccessZones: Total: $CLUSTER_INFO{'Access Zones'}"); last; } } if (!$CLUSTER_INFO{'Access Zones'}) { if (isCurrentOneFS('<7.1.1') and $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.1.1") >= 0) { warning("Unable to verify the number of Access Zones, manually check if there is more than just the System Access Zone configured"); } } elsif ($CLUSTER_INFO{'Access Zones'} > 1 and $RPS{'Target'} and isCurrentOneFS('<7.1.1') and compareVersions($RPS{'Target'}, "7.1.1") >= 0) { checkOverlap(); checkNonSystemPools(); warning("We strongly recommend that you read the OneFS Upgrade Planning and Process Guide and contact EMC Isilon Technical Support before upgrading the cluster to OneFS 7.1.1 or later. This will help to ensure that changes to access zones will not interrupt your workflow."); addCustomerInformation("Access Zones", @Logger::WARN, @Logger::FAIL); } info("Refer to KB 335112 (https://www.dell.com/support/kbdoc/335112) for details.") if isResults(&WARN); # Output Check result if ($CLUSTER_INFO{'Access Zones'}) { printResults($CLUSTER_INFO{'Access Zones'}); } else { printResults(); } } # End checkAccessZones $CHECKS{'checkDestinationOneFS'}{'Description'} = 'Checks destination OneFS version'; ### # Perform steps to 'adjust' the target version and verify it is a valid target sub checkDestinationOneFS { if (wasCalled()) { return } # FCO F010517EE and not already at an FCO required OneFS version if ($RPS{'F010517EE'} and !isDestinationOneFS() and ((isCurrentOneFS(qr/^7/) and isCurrentOneFS('<7.2.1.6')) or (isCurrentOneFS(qr/^8/) and isCurrentOneFS('<8.0.0.4')) or isCurrentOneFS('8.0.1.0'))) { # No destination OneFS version provided if (isCurrentOneFS(qr/8\.0\.1\.0/)) { $RPS{'Target'} = '8.1.2.0'; } else { $RPS{'Target'} = '8.0.0.7'; } warning('No Destination OneFS version provided for FCO F010517EE,'. ' setting destination to recommended, OneFS '. isDestinationOneFS()); } # FCO F033018EE but no selected target and running OneFS 8.1.2.0 or lower if ($RPS{'F033018EE'} and !isDestinationOneFS() and isCurrentOneFS('<8.1.2')) { $RPS{'Target'} = '8.1.2.0'; warning('No Destination OneFS version provided for FCO F033018EE,'. ' setting destination to recommended, OneFS '. isDestinationOneFS()); } # FCO F082318EE but no selected target and running OneFS 8.1.0.4 or lower if ($RPS{'F082318EE'} and !isDestinationOneFS() and isCurrentOneFS('<=8.1.0.4')) { $RPS{'Target'} = '8.1.2.0'; warning('No Destination OneFS version provided for FCO F082318EE,'. ' setting destination to recommended, OneFS '. isDestinationOneFS()); } # TSE T102518EE but no selected target and running OneFS 8.1.0.4 or lower if ($RPS{'T102518EE'} and !isDestinationOneFS() and isCurrentOneFS('<=8.1.0.4')) { $RPS{'Target'} = '8.1.2.0'; warning('No Destination OneFS version provided for TSE T102518EE,'. 'setting destination to recommended, OneFS '. isDestinationOneFS()); } # TSE T102518EE but OneFS 8.1.0.4 selected as destination OneFS if ($RPS{'T102518EE'} and isDestinationOneFS() and isDestinationOneFS('<=8.1.0.4')) { warning('Destination OneFS version '.isDestinationOneFS(). ' meets TSE requirements for TSE T102518EE, but OneFS 8.1.2.0'. ' and later is recommended.'); } # TSE T062519EE but no destination OneFS if ($RPS{'T062519EE'} and !isDestinationOneFS() and isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11) { $RPS{'Target'} = '8.2.0.0'; warning('No Destination OneFS version provided for TSE T062519EE, '. 'setting destination to recommended, OneFS '. isDestinationOneFS()); } # PFN ET003823, but no destination OneFS if ($RPS{'ET003823'} and !isDestinationOneFS() and isCurrentOneFS('8.2.1.0') and $CLUSTER_INFO{'Build'} < 4) { $RPS{'Target'} = '8.2.1.0'; warning('No Destination OneFS version provided for PFN ET003823, '. 'setting destination to recommended, OneFS '. isDestinationOneFS()); } # Removes the destination OneFS version if it is less than the current if (isDestinationOneFS()) { if (isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11 and isDestinationOneFS('8.2.0.0')) { # Allow upgrade to 8.2.0 build 11 from 8.2.0 build 9 } elsif (isCurrentOneFS('8.2.1.0') and $CLUSTER_INFO{'Build'} < 4 and isDestinationOneFS('8.2.1.0')) { # Allow upgrade to 8.2.1 build 3 from 8.2.1 build 4 } elsif (isCurrentOneFS('>='.isDestinationOneFS())) { # Remove the destination OneFS version if it is less than or equal # to the current OneFS version warning('Destination OneFS version '.isDestinationOneFS(). ' is less than or equal to the current OneFS version,'. ' removing destination OneFS version'); delete $RPS{'Target'}; } } # Recommend the latest patch releases for OneFS 9.1+ if (isDestinationOneFS('>=9.1') or isCurrentOneFS('>=9.1')) { our %Patch_Releases; my $rel = isDestinationOneFS()?isDestinationOneFS():isCurrentOneFS(); my ($major, $minor, $service) = split(/\./, $rel); my $re_branch = "^$major\.$minor\.$service\."; my @destinations = grep { /$re_branch/ } keys %Patch_Releases; debug("destinations = [@destinations], re_branch = $re_branch"); my ($latest) = sort {compareVersions($b, $a)} @destinations; debug("latest = $latest"); if ($latest) { # Avoid 9.2.0.0/9.2.1.0/9.3.0.0 unpatched releases at all costs my $SPF_92 = 'Due to critical bugs seen in OneFS %s the'. ' destination OneFS version has been automatically adjusted to'. ' OneFS %s'; if (isDestinationOneFS('9.2.1.0') or isDestinationOneFS('9.2.0.0') or isDestinationOneFS('9.3.0.0')) { addCustomerInformation( 'Upgrade Plan', critical(sprintf($SPF_92, isDestinationOneFS(), $latest))); $RPS{'Target'} = $latest; } if (isDestinationOneFS('>=9.1') and isDestinationOneFS("<$latest")) { warning('There is a newer patch release available for OneFS'. " $major.$minor.$service: $latest"); } elsif (!isDestinationOneFS() and isCurrentOneFS('>=9.1') and isCurrentOneFS("<$latest")) { warning('There is a newer patch release available for OneFS'. " $major.$minor.$service: $latest"); } } } # Identify F600/F900 nodes running/upgrading to a version less than 9.1.0.5 gatherProductInfo(); my $SPF_NVME = 'OneFS 9.1.0.5 and later releases contain critical updates'. ' for %s nodes'; my %nvme_types = (); foreach my $lnn (@LNNS) { # Node Product details must be available if (!$NODE_INFO{$lnn}{'Product'}) { next } # Identify F600/F900 NVMe nodes if ($NODE_INFO{$lnn}{'Product'} =~ /^(F[69]00)/) { $nvme_types{$1} = 1 } } if (%nvme_types) { my $msg = sprintf($SPF_NVME, join(' and ', keys %nvme_types)); if (!isDestinationOneFS() and isCurrentOneFS('<9.1.0.5') and !hasPatch('9.1.0_KDA-CUSTOM_2021-02_PSP-945') or isDestinationOneFS('<9.1.0.5')) { addCustomerInformation(critical($msg)); } } # Display destination OneFS version and any messages if (isDestinationOneFS()) { printCheck('Destination OneFS Version', isDestinationOneFS()); } if (isResults(&INFO)) { printCheck('Destination OneFS Version'); printResults(); } } # End checkDestinationOneFS $CHECKS{'checkDiskpools'}{'Description'} = 'Checks diskpools and class equivalence for OneFS upgrades going across 7.0.'; sub checkDiskpools { if (wasCalled()) { return } # Only check for upgrades from pre to 7.0 to post 7.0. if (!$RPS{'Target'} or isCurrentOneFS('>=7') or compareVersions($RPS{'Target'}, "7") < 0) { debug("checkDiskpools: Check skipped, only applies when crossing OneFS 7.0"); return; } printCheck("Diskpools"); gatherIsiStatPools(); gatherProductInfo(); # Populate an equivalence class stored in NODE_INFO{$node}{'Class'} # based on node product information in NODE_INFO{$node}{'Product'} # Class 0 means there is no equivalence class. foreach my $node (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$node}{'Product'}; $_ = $NODE_INFO{$node}{'Product'}; if (/IQ 6000x/i or /^X200.*6144MB.*6144GB$/i) { $NODE_INFO{$node}{'Class'} = 1; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 12000x/i or /^X200.*6144MB.*12TB$/i) { $NODE_INFO{$node}{'Class'} = 2; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 36000x/i or /^X400.*24GB.*36TB$/i) { $NODE_INFO{$node}{'Class'} = 3; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 72000x/i or /^X400.*24GB.*72TB$/i) { $NODE_INFO{$node}{'Class'} = 4; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 32000X-SSD/i or /^X400.*24GB.*-32TB.*SSD$/i) { $NODE_INFO{$node}{'Class'} = 5; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 36NL/i or /^NL400.*12GB.*-36TB$/i) { $NODE_INFO{$node}{'Class'} = 6; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 72NL/i or /^NL400.*12GB.*-72TB$/i) { $NODE_INFO{$node}{'Class'} = 7; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } elsif (/IQ 108NL/i or /^NL400.*12GB.*-108TB$/i) { $NODE_INFO{$node}{'Class'} = 8; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } else { $NODE_INFO{$node}{'Class'} = 0; debug("NODE_INFO->${node}->Class=$NODE_INFO{$node}{'Class'}"); } } # Iterate through and compare equivalence classes for each node # using NODE_INFO{$node}{'Pool'} and NODE_INFO{$node}{'Class'} foreach my $nodea (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$nodea}{'Pool'}; next unless $NODE_INFO{$nodea}{'Class'}; foreach my $nodeb (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$nodeb}{'Class'}; next unless $NODE_INFO{$nodeb}{'Pool'}; # Move to the next node if we've already done the comparison. # I.E.: Node A compares against every node greater than it on it's # first path. So if the node b is less than the node a, it's already # been analyzed. next if (compareVersions($nodea, $nodeb) >= 0); # Node a and node b are in equivalent classes, but they are not in the same diskpool. if ($NODE_INFO{$nodea}{'Class'} and $NODE_INFO{$nodeb}{'Class'} and $NODE_INFO{$nodea}{'Class'} == $NODE_INFO{$nodeb}{'Class'} and $NODE_INFO{$nodea}{'Pool'} ne $NODE_INFO{$nodeb}{'Pool'}) { fail("Nodes $nodea and $nodeb are of equivalent type, and need to be in the same diskpool"); } # Node a and node b are in the same diskpool, and either # they are not in the same equivalence class or they are different products. if ($NODE_INFO{$nodea}{'Pool'} eq $NODE_INFO{$nodeb}{'Pool'} and ($NODE_INFO{$nodea}{'Class'} != $NODE_INFO{$nodeb}{'Class'} or (!$NODE_INFO{$nodea}{'Class'} and !$NODE_INFO{$nodeb}{'Class'} and $NODE_INFO{$nodea}{'Product'} ne $NODE_INFO{$nodeb}{'Product'}))) { fail("Nodes $nodea and $nodeb are not of equivalent type, and need to be in different diskpools"); } } } # Output Check result printResults(); } # End checkDiskPools ### # checkFilepoolPolicies: Checks filepools for final match being set and # names starting with a number. $CHECKS{'checkFilepoolPolicies'}{'Description'} = "Checks GNA requirements and checks filepools for final match being set and names starting with a number."; sub checkFilepoolPolicies { if (wasCalled()) { return } if (isCurrentOneFS('>=7.1.1')) { debug("checkFilepoolPolicies: Skipping. Current version greater than 7.1.1"); return; } elsif (!$RPS{'Target'}) { debug("checkFilepoolPolicies: Skipping. No target version"); return; } printCheck("Filepool Policies"); gatherForceSpill(); # Gather information debug("checkFilepoolPolicies: Gathering filepool policy information"); my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig smartpools.filepools]); } else { $gather = gatherLocal(qq[2>&1 grep -h 'smartpools.filepools' local/isi_gconfig local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]); } # Analyze for issues my $policy_name = ""; my $gna_enabled = 0; my @policies_with_final_match = (); my @policies_start_with_number = (); debug("checkFilepoolPolicies: Searching for filepool policy related issues"); foreach (@$gather) { if (/smartpools\.filepools\.policies\.(\d+) /i) { $policy_name = ""; } elsif (/smartpools\.filepools\.policies\.\d\.name .*\"(.*)\"/i) { $policy_name = $1; if ($policy_name =~ /^\d/) { unless (grep {$_ eq $policy_name} @policies_start_with_number) { push(@policies_start_with_number, $policy_name); debug("checkFilepoolPolicies: !!! Policy name begins with a number: $policy_name"); } } } elsif (/smartpools\.filepools.*final_match.*false/i) { unless (grep {$_ eq $policy_name} @policies_with_final_match) { push(@policies_with_final_match, $policy_name); debug("checkFilepoolPolicies: !!! Policy with final match set to false: $policy_name"); } } elsif (/smartpools\.filepools\.gna_enabled.*true/i) { debug("checkFilepoolPolicies: Found GNA is enabled"); $gna_enabled = 1; } } # Policies start with a number if (isCurrentOneFS('<7.1.1') and compareVersions($RPS{'Target'}, "7.1.1") >= 0 and @policies_start_with_number) { fail("Upgrade would invalidate all file pool policies with names that start with a number. As of OneFS 7.1.1, filepool policy names can no longer have a numeric first character. The following policies must be renamed before upgrade: ".join(",",@policies_start_with_number)); fail("Upgrade would invalidate a file pool policy that doesn't stop matching. As of OneFS 7.1.1, filepool policy matching terminates after a first match is found. The following polices must be rewritten before upgrade: ".join(",",@policies_with_final_match)); } # GNA enabled if ($gna_enabled) { # Determine the number of nodes with SSD storage not used as L3 cache my @nodes_without_ssd = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $NODE_INFO{$lnn}{'SSD Storage'}; next unless $NODE_INFO{$lnn}{'SSD Storage'} =~ /No SSDs|No Storage SSDs|Diskless|L3/; push(@nodes_without_ssd, $lnn); } # Calculate the percent of nodes which are being used for SSD storage my $percent_ssd_nodes = 0; if(@nodes_without_ssd) { $percent_ssd_nodes = 100 * ($CLUSTER_INFO{'Node Count'} - scalar(@nodes_without_ssd))/$CLUSTER_INFO{'Node Count'}; debug("checkFilepoolPolicies: Percent SSD Nodes: $percent_ssd_nodes"); } # A minimum of 20% must be SSD storage nodes not used for L3 cache if ($percent_ssd_nodes < 20) { warning("Cluster is enabled for GNA but only ${percent_ssd_nodes}% of all nodes have SSD storage, the minimum requirement is 20%"); } # Calculate the percent of storage which is SSD storage my $percent_ssd_storage = 0; if ($CLUSTER_INFO{'SSD Size'} and $CLUSTER_INFO{'HDD Size'}) { $percent_ssd_storage = sprintf("%.1f", 100 * $CLUSTER_INFO{'SSD Size'}/$CLUSTER_INFO{'HDD Size'}); debug("checkFilepoolPolicies: Percent SSD Storage: $percent_ssd_storage"); } # A minimum of 1.5% of storage must be SSD storage if ($percent_ssd_storage < 1.5) { warning("Cluster is enabled for GNA but only ${percent_ssd_storage}% of all storage is SSD storage, the minimum requirement is 1.5%"); } # Flag an issue for upgrades from prior to 7 to 7 or later where GNA would be deactivated as part of the upgrade if (isCurrentOneFS('<7') and compareVersions($RPS{'Target'}, "7") >= 0 and ($percent_ssd_nodes < 20 or $percent_ssd_storage < 1.5)) { fail("Cluster does not meet minimum requirements for Global Namespace Acceleration and an upgrade would deactivate Global Namespace Acceleration. Either disable GNA, or call Isilon support. Important: do NOT bypass this check if you are also failing the CheckDiskpoolsMembership check or any other upgrade check.") if $percent_ssd_storage < 1.5; } # Warn if force spill over is enabled which could be bad warning("Disk Pools are configured to spill over and write data to SSD drives."); } addCustomerInformation($_) foreach (@Logger::FAIL); printResults(); } # End checkFilepoolPolicies sub checkFileSystemFormatVersion { return if wasCalled(); my %gather = (); if ($RPS{'Live'}) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array sysctl efs.gmp.fs_fmt_version"); } else { $gather = gatherCluster("2>&1 grep -H fs_fmt_version */efs.gmp.current_info"); } my @zero_fs_fmt_version = (); my $fs_fmt_version_odd = 0; verboseHeader("checkFileSystemFormatVersion"); verbose(sprintf("%5s %5s", "LNN", "fs_fmt_version")); foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /^\s{0,8}fs_fmt_version\s*=\s*(\d+)$/ or $line =~ /fs_fmt_version:\s*(\d+)$/; my $fs_fmt_version = $1; verbose(sprintf("%5s %5s", $lnn, $fs_fmt_version)); push(@zero_fs_fmt_version, $lnn) if ($fs_fmt_version eq "0"); $fs_fmt_version_odd = 1 if ($fs_fmt_version % 2 == 1); } } # Flag if fs_fmt_version is zero on any nodes if (@zero_fs_fmt_version) { fail("The following nodes are reporting a fs_fmt_version of 0: ". compressRange(@zero_fs_fmt_version)); } # Flag if fs_fmt_version is odd on any nodes if ($fs_fmt_version_odd) { if ($RPS{'Target'}) { fail("An odd fs_fmt_version was found, this is indicative that the Upgrade job has not completed and needs to complete prior to an upgrade to OneFS."); } else { warning("An odd fs_fmt_version was found, this is indicative that the Upgrade job has not completed and needs to complete prior to an upgrade to OneFS."); } } } # End checkFileSystemFormatVersion ### # checkKerberos8000: Checks for an issue with the Kerberos # configuration file for OneFS upgrades targetting 8.0.0.0. $CHECKS{'checkKerberos8000'}{'Description'} = "Checks for an issue with the Kerberos configuration file for OneFS upgrades targetting 8.0.0.0."; sub checkKerberos8000 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.0')) { debug("checkKerberos8000: Skipping. Current version already greater than 8.0.0.0."); return; } elsif (!$RPS{'Target'} or compareVersions($RPS{'Target'}, "8.0.0.0") != 0) { debug("checkKerberos8000: Skipping. Target version not 8.0.0.0."); return; } printCheck("Kerberos & OneFS 8.0.0.0"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 cat /etc/likewise-krb5-ad.conf"); } else { $gather = gatherLocal("2>&1 cat local/ifsvar_etc.tar/etc/likewise-krb5-ad.conf"); } # Analyze information for issues. my $missing_permitted_enctypes = 1; foreach my $line (@$gather) { debug("checkKerberos8000: $line"); if ($line =~ /permitted_enctypes/) { $missing_permitted_enctypes = 0; debug("Found permitted_enctypes, not affected"); } } # If missing the permitted_enctypes line if ($missing_permitted_enctypes) { fail("Permitted Encryption types must be added to /etc/likewise-krb5-ad.conf prior to a OneFS upgrade to 8.0.0.0.", "Escalate to support and reference Bug 168999 or complete additional pre-upgrade procedures to address the issue.") } # Output Check result printResults(); } # End checkKerberos8000 ### # checkLastZoneID8000: Checks for gaps in access zones that may cause # major issues for OneFS upgrades targeting 8.0.0.0. $CHECKS{'checkLastZoneID8000'}{'Description'} = "Checks for gaps in access zones that may cause major issues for OneFS upgrades targeting 8.0.0.0."; sub checkLastZoneID8000 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.0') >= 0) { debug("checkLastZoneID8000: Skipping. Current version already greater than 8.0.0.0."); return; } elsif (!$RPS{'Target'} or compareVersions($RPS{'Target'}, "8.0.0.0") != 0) { debug("checkLastZoneID8000: Skipping. Target version not 8.0.0.0"); return; } checkAccessZones(); printCheck("LastZoneID & OneFS 8.0.0.0"); my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 cat /ifs/.ifsvar/main_config.gc"); } else { $gather = gatherLocal("2>&1 cat local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } # Default lastzoneid to 1 for cases where they have never added/deleted any access zones my $lastzoneid = 1; foreach (@$gather) { $lastzoneid = $1 if (/.*lastzoneid.*\"(.*)\"/i); } # If the total number of access zones does not match the lastzoneid addCustomerInformation(fail("SMB share and NFS export configuration may be lost after an upgrade to 8.0.0.0, review KB480190 for details.")) if ($CLUSTER_INFO{'Access Zones'} and $CLUSTER_INFO{'Access Zones'} ne $lastzoneid); # Output Check result printResults(); } # End checkLastZoneID8000 ### # checkLegacyLDAP: Checks for Legacy LDAP being enabled in OneFS 6 to OneFS 7 upgrades $CHECKS{'checkLegacyLDAP'}{'Description'} = "Checks for Legacy LDAP being enabled in OneFS 6 to OneFS 7 upgrades"; sub checkLegacyLDAP { if (wasCalled()) { return } return unless ($RPS{'Target'} and isCurrentOneFS('<7') and compareVersions($RPS{'Target'}, "7") >= 0); printCheck("Legacy LDAP"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 cat /etc/mcp/override/ldap.xml"); } else { $gather = gatherLocal("2>&1 cat local/ifsvar_etc.tar/etc/mcp/override/ldap.xml"); } # Analyze information for issues. foreach my $line (@$gather) { fail("Legacy LDAP cannot be upgraded. Please migrate all settings under 'Legacy LDAP' in the WebUI to the 'LDAP' tab and validate correct LDAP functionality before upgrading.") if ($line =~ /modify-attribute.*id="ldap".*attribute="enable".*value="1"/); } addCustomerInformation("Legacy LDAP cannot be upgraded. Please migrate all settings under 'Legacy LDAP' in the WebUI to the 'LDAP' tab and validate correct LDAP functionality before upgrading.") if isResults(&FAIL); # Output Check result printResults(); } # End checkLegacyLDAP sub checkNFS8004 { if (wasCalled()) { return } if ($RPS{'Target'}) { return unless $RPS{'Target'} eq "8.0.0.4"; checkUpgradePath(); my $current_code = $CLUSTER_INFO{'OneFS'}; my $affected = 0; foreach my $target_code (sort {compareVersions($a, $b)} @{$RPS{'Upgrade Path'}}) { if (compareVersions($current_code, "7.2") == 0 and compareVersions($target_code, "8.0.0.4") == 0) { $affected = 1; } $current_code = $target_code; } return unless $affected; } else { return unless isCurrentOneFS('8.0.0.4'); } gatherNFSExport(); return unless $CLUSTER_INFO{'NFS Export'}; # Post upgrade issue, NFS Export has - as the Security Type my @blank_export_ids = (); my @unix_export_ids = (); foreach my $id (keys %{$CLUSTER_INFO{'NFS Export'}}) { next unless $CLUSTER_INFO{'NFS Export'}{$id}{'Security Type'}; if ($CLUSTER_INFO{'NFS Export'}{$id}{'Security Type'} =~ /^\s*-\s*$/) { push(@blank_export_ids, $id); debug("checkNFS8004: Export ID $id impacted!"); } if ($CLUSTER_INFO{'NFS Export'}{$id}{'Security Type'} =~ /^\s*unix\s*$/) { push(@unix_export_ids, $id); debug("checkNFS8004: Export ID $id is unix."); } } if (@blank_export_ids and isCurrentOneFS('8.0.0.4')) { fail("The following NFS Export IDs are impacted by KB496341: ".join(",",@blank_export_ids)); fail("Review the KB article and complete resolution steps for Post upgrade on 8.0.0.4"); } if (@unix_export_ids and isCurrentOneFS('<8')) { warning("The following NFS Export IDs have unix Security Type, may be impacted by KB496341: ".join(",",@unix_export_ids)); warning("Prior to an upgrade to 8.0.0.4, review the KB article and complete pre-upgrade resolution steps"); addRequiredFiles("NFS Export Security Repair Script"); } } # End checkNFS8004 sub checkNonSystemPools { if (wasCalled()) { return } return unless $RPS{'Target'} and compareVersions($RPS{'Target'}, "7.1.1") >= 0 and isCurrentOneFS('<7.1.1'); gatherFlexNet(); my $non_system_zone_pools = 0; foreach my $id (keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}) { next unless $CLUSTER_INFO{'FlexNet'}{'Pools'}{$id}{'Auth Zone'}; if ($CLUSTER_INFO{'FlexNet'}{'Pools'}{$id}{'Auth Zone'} ne 1) { # Flag issue $non_system_zone_pools = 1; debug("checkNonSystemPools: !!! Network pool $id configured with access zone $CLUSTER_INFO{'FlexNet'}{'Pools'}{$id}{'Auth Zone'}"); } } if ($non_system_zone_pools) { warning("NFS clients accessing the cluster via any non-System access zone network pools will experience disruption during the upgrade."); warning("NFS exports will default to the System access zone after upgrading to OneFS 7.2 or later."); warning("NFS access is only available via System access zone network pools in OneFS 7.1.1."); warning("CLI and WebUI administration is only available via System access zone network pools starting in OneFS 7.1.1."); warning("HTTP and FTP access is only available via System access zone network pools starting in OneFS 7.1.1."); } } # End checkNonSystemPools $CHECKS{'checkOneFSVersions'}{'Description'} = "Checks running version and target version for any issues. Failure: Any version mismatch across all nodes"; sub checkOneFSVersions { if (wasCalled()) { return } printCheck("OneFS Version"); verboseHeader("OneFS Versions"); my $spf = "%-5s %-12s"; verbose(sprintf($spf, "LNN", "OneFS")); verbose(sprintf($spf, "-"x5, "-"x12)); my %onefs_versions = (); my %OneFSs = (); my %current_onefs_versions = (); my $commit_mismatch = 0; foreach my $lnn (@LNNS) { verbose(sprintf($spf, $lnn, $NODE_INFO{$lnn}{'OneFS'})) if $lnn and $NODE_INFO{$lnn}{'OneFS'}; # Compile a list of OneFS versions as keys in a hash if ($NODE_INFO{$lnn}{'onefs_version'}) { $onefs_versions{$NODE_INFO{$lnn}{'onefs_version'}} = 1; if ($NODE_INFO{$lnn}{'committed_version'} and $NODE_INFO{$lnn}{'committed_version'} ne $NODE_INFO{$lnn}{'onefs_version'}) { $commit_mismatch = 1; } } $current_onefs_versions{$NODE_INFO{$lnn}{'current_onefs_version'}} = 1 if $NODE_INFO{$lnn}{'current_onefs_version'}; $OneFSs{$NODE_INFO{$lnn}{'OneFS'}} = 1 if $NODE_INFO{$lnn}{'OneFS'}; } # Fail if multiple OneFS versions found if (scalar(keys %OneFSs) > 1) { fail("Multiple OneFS versions detected: ".join(", ", keys %OneFSs)); } elsif (scalar(keys %current_onefs_versions) > 1) { fail("Multiple OneFS versions detected: ".join(", ", keys %current_onefs_versions)); } elsif (scalar(keys %onefs_versions) > 1) { fail("Multiple OneFS versions detected: ".join(", ", keys %onefs_versions)); } # Fail if commit version differs from node versions if ($commit_mismatch) { fail("Committed OneFS version does not match node OneFS versions"); } # Beta versions if ($CLUSTER_INFO{'Beta'}) { if ($RPS{'Target'}) { fail("BETA version detected, OneFS upgrades are not supported from BETA versions, destination OneFS version removed"); delete $RPS{'Target'}; } else { warning("BETA version detected, OneFS upgrades are not supported from BETA versions"); } } if ($RPS{'F010517EE'}) { if ((isCurrentOneFS(qr/^7/) and isCurrentOneFS('<7.2.1.6')) or (isCurrentOneFS(qr/^8/) and isCurrentOneFS('<8.0.0.4')) or (isCurrentOneFS(qr/^8\.0\.1\.0/))) { $RPS{'FCO'}{'F010517EEA'}{'Severity'} = "INFO"; @{$RPS{'FCO'}{'F010517EEA'}{'Message'}} = ("Current OneFS $CLUSTER_INFO{'OneFS'} does not meet requirements for FCO F010517EE"); } } if (isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11 and !$RPS{'Target'}) { # Encourage an upgrade to 8.2.0 build 011 from 8.2.0 build 009 critical("OneFS 8.2.0 build $CLUSTER_INFO{'Build'} was detected, an upgrade to OneFS 8.2.0 build 011 or later is recommended as monthly RUP patches will not be developed for build $CLUSTER_INFO{'Build'}."); } if (isCurrentOneFS("8.2.1.0") and $CLUSTER_INFO{'Build'} < 4 and !$RPS{'Target'}) { # Encourage an upgrade to 8.2.1 build 004 from 8.2.1 build 003 critical("OneFS 8.2.1 build $CLUSTER_INFO{'Build'} was detected, an upgrade to OneFS 8.2.1 build 004 or later is recommended as monthly RUP patches will not be developed for build $CLUSTER_INFO{'Build'}."); } # Refer to KB for details info("Refer to KB 335096 (https://www.dell.com/support/kbdoc/335096) for details.") if isResults(&WARN); # Output results printResults(); } # End checkOneFSVersion sub checkOverlap { if (wasCalled()) { return } unless ($RPS{'Target'} and isCurrentOneFS('>=7') and isCurrentOneFS('<7.1.1') and compareVersions($RPS{'Target'}, "7.1.1") >= 0 and compareVersions($RPS{'Target'}, "8") < 0) { debug("checkOverlap: Skipped.."); return; } # Gather SMB Share information from cluster or from logset debug("checkOverlap: gathering SMB information"); my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi_classic smb dump"); } else { $gather = gatherLocal("2>&1 cat local/isi_smb"); } # Populate a hash of share name -> share path my %SMB_Shares = (); my ($share_path, $share_name) = ""; foreach (@$gather) { last if /SMB Share Permissions:/; $share_name = $1 if /^Name:\s*(\S.*)/ or /^\s*Share Name:\s*(\S.*)/; $share_path = $1 if /^Path:\s*(\S.*)/ or /^\s*Path:\s*(\S.*)/; next unless $share_name and $share_path; $SMB_Shares{$share_name} = $share_path; ($share_name, $share_path) = ""; } # Gather Access Zone information debug("checkOverlap: gathering Access Zone information"); $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 cat /ifs/.ifsvar/main_config.gc"); } else { $gather = gatherLocal("2>&1 cat local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } # Populate an array of SMB shares associated with each zone my $zone_name = ""; foreach (@$gather) { next if !/Zones/; $zone_name = $1 if /\.Zones\._key\.[^\.]*\._name.*\"(.*)\"/; next unless $zone_name; if (/ShareList\.value .*?\"\[ (.*) \]\"/ and !$CLUSTER_INFO{'Zones'}{$zone_name}{'All SMB Shares'}) { my $sharelist = $1; $sharelist =~ s/\\\"//g; my @sharelist = split(", ", $sharelist); foreach (@sharelist) { my $share_name = $1 if /^(.*?):/; next unless $share_name and $SMB_Shares{$share_name}; push(@{$CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}}, $share_name); debug("CLUSTER_INFO->Zones->${zone_name}->Share List=$share_name"); } } if (/AllSMBShares\.value .* \"1\"/) { $CLUSTER_INFO{'Zones'}{$zone_name}{'All SMB Shares'} = 1; debug("CLUSTER_INFO->Zones->${zone_name}->All SMB Shares=1"); @{$CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}} = keys %SMB_Shares; } } debug("checkOverlap: For each access zone, find the lowest common path for the SMB shares associated with it."); my @lowest_common_paths = (); foreach my $zone_name (keys %{$CLUSTER_INFO{'Zones'}}) { next if $zone_name eq "System"; debug("checkOverlap: findLowestCommonPath: $zone_name"); $CLUSTER_INFO{'Zones'}{$zone_name}{'Lowest Common Path'} = findLowestCommonPath(@SMB_Shares{@{$CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}}}) if ($CLUSTER_INFO{'Zones'}{$zone_name}{'Share List'}); push(@lowest_common_paths, "$zone_name -> ".$CLUSTER_INFO{'Zones'}{$zone_name}{'Lowest Common Path'}) if ($CLUSTER_INFO{'Zones'}{$zone_name}{'Lowest Common Path'}); } info("Lowest common SMB share paths for each Access Zone are as follows:", @lowest_common_paths) if @lowest_common_paths; debug("checkOverlap: Check each zones shares against every other zones shares for nested shares"); my $nested_share = 0; my $shared_share = 0; foreach my $zone_a (keys %{$CLUSTER_INFO{'Zones'}}) { next if $zone_a eq "System"; foreach my $zone_b (keys %{$CLUSTER_INFO{'Zones'}}) { next if $zone_b eq "System" or $zone_a eq $zone_b; foreach my $share_a (@{$CLUSTER_INFO{'Zones'}{$zone_a}{'Share List'}}) { foreach my $share_b (@{$CLUSTER_INFO{'Zones'}{$zone_b}{'Share List'}}) { if ($SMB_Shares{$share_a} eq $SMB_Shares{$share_b}) { $shared_share = 1; debug("checkOverlap: !!! $share_a associated with $zone_a and $zone_b"); } elsif ($SMB_Shares{$share_a} =~ /^$SMB_Shares{$share_b}/) { $nested_share = 1; debug("checkOverlap: !!! ${share_a}:$SMB_Shares{$share_a} in $zone_a nested within ${share_b}:$SMB_Shares{$share_b} of $zone_b"); } } } } } if ($nested_share or $shared_share) { warning("Detected one or more SMB shares associated with multiple access zones where the share paths are equal or nested. This configuration is no longer supported. Upgrade will succeed, but you will be unable to create new access zones until the path overlap is fixed."); } } # End checkOverlap ### # checkSBR8000: Checks if SBR is enabled pre-upgrade for OneFS upgrades # targeting 8.0.0.0/1. $CHECKS{'checkSBR8000'}{'Description'} = "Checks if SBR is enabled pre-upgrade for OneFS upgrades targeting 8.0.0.0/1."; sub checkSBR8000 { return if wasCalled(); if (isCurrentOneFS('>=8.0.0.2')) { debug("checkSBR8000: Skipping. Current version greater than or equal to 8.0.0.2."); return; } elsif (isCurrentOneFS('<7.2')) { debug("checkSBR8000: Skipping. Current version less than 7.2."); return; } elsif (!$RPS{'Target'} or $RPS{'Target'} !~ /^8.0.0.[01]$/) { debug("checkSBR8000: Skipping. Target version not 8.0.0.0 or 8.0.0.1."); return; } printCheck("SBR & OneFS 8.0.0.0/1"); # Gather information from cluster or from logset gatherFlexNet(); # If Source Based Routing is in use. if ($CLUSTER_INFO{'FlexNet'}{'SBR'}) { fail("Source Based Routing is enabled. An issue exists in 8.0.0.0 and 8.0.0.1 where when source based routing is enabled, nodes will not reply outside of their subnet. This could potentially leave the cluster in an unreachable state. It is not recommended to upgrade to either of these versions with SBR enabled. This issue is due to be fixed in the release of 8.0.0.2. Alternatively, if upgraded to 8.0.0.0 or 8.0.0.1, a fix is available in patch-169853."); } # Output Check result printResults(); } # End checkSBR8000 sub checkSupportabilityIsilonSD { if (wasCalled()) { return } gatherProductInfo(); if (!$CLUSTER_INFO{'IsilonSD'}) { return } my $CMD_LOGS = q[grep ^model local/ifsvar_modules.tar/modules/vonefs/gconfig/vonefs_config.gc]; my $CMD_LIVE = qq[${SUDO} isi_gconfig -t vonefs_config | grep ^model]; my $MSG_SUPPORT = 'With a FREE license, you are not entitled to any support from Isilon for product-related questions.'; my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 $CMD_LIVE ]); } else { $gather = gatherLocal(qq[2>&1 $CMD_LOGS ]); } my $free_edition = 0; foreach my $line (@$gather) { my ($key, $value) = parseGconfigItem($line); if ($key != 'model') { next } if ($value =~ /free/i) { $free_edition = 1 } } if ($free_edition) { if ($RPS{'UpgradePlan'}) { fail($MSG_SUPPORT); } else { warning($MSG_SUPPORT); } } } sub checkSupportabilityNodeType { if (wasCalled()) { return } gatherProductInfo(); my $SPF_EOSS = 'The following nodes are EOSS: %s'; my $MSG_SIMULATOR = 'Isilon OneFS simulators are not entitled to any support from Isilon for product-related questions.'; my @eoss = (); my $simulator = 0; foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'Product'}) { next } if ($simulator) { last } if ($NODE_INFO{$lnn}{'Product'} =~ /SIMULATOR/i) { $simulator = 1 } if ($NODE_INFO{$lnn}{'Product'} =~ /^IQ/i) { push(@eoss, $lnn) } if ($NODE_INFO{$lnn}{'Product'} =~ /^(?:S200|NL400|X200|X400)/i) { push(@eoss, $lnn); } } if (@eoss) { warning(sprintf($SPF_EOSS, compressRange(@eoss))) } if ($simulator) { if ($RPS{'UpgradePlan'}) { fail($MSG_SIMULATOR); } else { warning($MSG_SIMULATOR); } } } sub checkSupportabilityOneFS { if (isCurrentOneFS('<=8.1')) { warning('Current OneFS version, '.isCurrentOneFS().', is EOSL'); } if (isDestinationOneFS('<=8.1')) { warning('Destination OneFS version, '.isDestinationOneFS().', is EOSL'); } } $CHECKS{'checkSupportability'}{'Description'} = 'Checks cluster hardware and software supportability'; sub checkSupportability { if (wasCalled()) { return } printCheck('Supportability'); my $MSG_DOC = 'For more information on Isilon product support life refer to https://support.emc.com/docu45445_Isilon-Product-Availability.pdf'; checkSupportabilityOneFS(); checkSupportabilityNodeType(); checkSupportabilityIsilonSD(); if (isResults(&INFO)) { info($MSG_DOC) } printResults(); } # End checkSupportability ### # checkUpgradePath: Checks where multiple jumps are required and provides specifics required. $CHECKS{'checkUpgradePath'}{'Description'} = "Checks where multiple jumps are required and provides specifics required."; sub checkUpgradePath { if (wasCalled()) { return } # If there is no target version, skip if (!isDestinationOneFS()) { return } checkPatches(); checkKB546604(); # Skip if not a OneFS upgrade if (!isOneFSUpgrade()) { return } printCheck('Upgrade Path'); # Generate an upgrade path @{$RPS{'Upgrade Path'}} = generateUpgradePaths(isCurrentOneFS(), isDestinationOneFS()); debug(join(', ', @{$RPS{'Upgrade Path'}})); # Fail if we couldn't find any path if (scalar(@{$RPS{'Upgrade Path'}}) <= 1) { fail("No found upgrade path from $CLUSTER_INFO{'OneFS'} to $RPS{'Target'}"); delete $RPS{'UpgradePlan'}; } # Warn if target code requires multiple jumps if (scalar(@{$RPS{'Upgrade Path'}}) >= 3) { warning("The requested destination version will require multiple upgrade jumps with the following recommended steps: ".join(" -> ", @{$RPS{'Upgrade Path'}})); addCustomerInformation("Upgrade Plan", "The requested destination version will require multiple upgrade jumps with the following recommended steps: ".join(" -> ", @{$RPS{'Upgrade Path'}})); } my ($source, $target) = ""; my $parallel_onefs_support = 0; my $parallel_nfp_support = 0; foreach (@{$RPS{'Upgrade Path'}}) { $target = $_; if ($target and $source) { $RPS{'OneFS Upgrade'}{$source}{'Target'} = $target; if (isUpgradeSimultaneous($source, $target)) { warning("The upgrade from $source to $target must be performed as a simultaneous upgrade"); addCustomerInformation("Upgrade Plan", "The upgrade from $source to $target must be performed as a simultaneous upgrade"); } if (isUpgradeRestripe($source, $target)) { warning("After the upgrade from $source to ${target}, an 'Upgrade' data restriping job will run."); addCustomerInformation("After the upgrade from $source to ${target}, an 'Upgrade' data restriping job will run."); } # For all upgrades from prior to OneFS 8.1 to OneFS 8.1 and later if (compareVersions($source, '8.1') < 0 and compareVersions($target, '8.1') >= 0) { addCustomerInformation(warning("After the upgrade from OneFS $source to OneFS $target, a one-time license conversion to a software license must be performed, and, following the license update, if using an SRS gateway for remote connectivity the SRS configuration on the cluster should be updated. A quick reference on how to do this is available at KB 525065 (https://www.dell.com/support/kbdoc/525065)")); } # Check if parallel is supported if (compareVersions($source, '8.2.2') >= 0) { $parallel_onefs_support = 1; } if (compareVersions($source, '8.2.0') >= 0) { $parallel_nfp_support = 1; } } $source = $target; } # Flag if requested parallel method is not supported my $MSG_ONEFS_PARALLEL = 'Parallel method for OneFS upgrades was requested, but is not available prior to OneFS 8.2.2 release, rolling method will be used'; if (exists $RPS{'OneFS'}{'Method'} and $RPS{'OneFS'}{'Method'} eq 'parallel' and !$parallel_onefs_support) { warning($MSG_ONEFS_PARALLEL); $RPS{'OneFS'}{'Method'} = 'rolling'; } # Warn if upgrading to 8.0.1.0 in regards to being simultaneous. if (isDestinationOneFS(qr/^8\.0\.1\.0/)) { warning("Upgrades to 8.0.1.0 should only be performed simultaneous to avoid disruptions caused by an issue identified in 8.0.1.0 code level that is addressed in a disruptive patch."); } # Inform customers/warn about OneFS 8.0.1 being withdrawn and recommend other branches. if (isDestinationOneFS(qr/^8\.0\.1\./)) { addCustomerInformation(warning("Consider instead performing a OneFS upgrade to the target in the OneFS 8.0.0 branch or to a OneFS 8.1 version.")); } # For all upgrades from OneFS 8.0 to OneFS 8.1, disable isi_drive_repurpose_d service during the upgrade. if (isCurrentOneFS(qr/^8\.0\./) and isDestinationOneFS(qr/^(8\.1\.0\.[012]|8\.1\.1\.0)$/)) { critical("During an upgrade from OneFS $CLUSTER_INFO{'OneFS'} to $RPS{'Target'} the isi_drive_repurpose_d service should be disabled to prevent an issue where drives incorrectly smartfail."); } # For all upgrades from prior to OneFS 8.2 to OneFS 8.2 and later if ($RPS{'Target'} and isCurrentOneFS('<8.2') and compareVersions($RPS{'Target'}, "8.2") >= 0) { addCustomerInformation(warning('The SmartConnect Service IP (SSIP) was never intended for client connectivity or WebUI/CLI administration. In OneFS 8.2 and later, this is strictly enforced and the SSIP is not usable for client or administrative CLI/WebUI access to the cluster. For more details please review KB 536650 [https://www.dell.com/support/kbdoc/536650 - OneFS 8.2: Smartconnect Service IP Address (SSIP/VIP) does not allow connections beyond DNS traffic.]')); if (hasLicense('HDFS')) { addCustomerInformation(warning('Check and ensure that all local user accounts on the Isilon cluster that are used for HDFS operations are enabled. Changes to behavior in OneFS 8.2 and later will cause HDFS operations to fail when using local user accounts that are disabled. For more details please review KB 543086 [https://www.dell.com/support/kbdoc/543086 - {Isilon} OneFS 8.2 HDFS Local Service Accounts need to be Enabled for identity]')); } } if ($RPS{'Target'} and $CLUSTER_INFO{'IsilonSD'}) { addCustomerInformation(info('Prior to performing a OneFS upgrade on an IsilonSD Edge cluster, make sure to review the IsilonSD Edge compatibility details available at: https://support.emc.com/docu44518_Isilon_Supportability_and_Compatibility_Guide_.pdf')); } printResults(); } # End checkUpgradePath ### # gatherForceSpill: Checks if force spill is enabled sub gatherForceSpill { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} grep -h spill /etc/mcp/override/sysctl.conf"); } else { $gather = gatherLocal("2>&1 grep -h spill local/ifsvar_etc.tar/etc/mcp/override/sysctl.conf"); } # Analyze information for issues. my $force_spill; foreach my $line (@$gather) { if ($line =~ /disk_pool_global_force_spill.*1$/ or $line =~ /disk_pool_global_spill.*1$/) { $CLUSTER_INFO{'Force Spill'} = 1; debug("CLUSTER_INFO->Force Spill=1"); last; } } } # End gatherForceSpill ### # gatherOneFS: Gathers OneFS version from node_status.gc sub gatherOneFS { return if wasCalled(); # Gather information my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster (qq[2>&1 ${SUDO} isi_for_array 'grep current_onefs_version /etc/ifs/node_status.gc']); } else { $gather = gatherCluster (qq[2>&1 grep -H current_onefs_version */gconfig_local.tar/etc/ifs/node_status.gc]); } # Assign variables foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; foreach (@{$gather->{$lnn}}) { my ($key, $value) = parseGconfigItem($_); next unless $key eq "version_info.current_onefs_version"; $value =~ s/L$//; $NODE_INFO{$lnn}{'current_onefs_version'} = $value; debug("NODE_INFO->${lnn}->current_onefs_version=$value"); } } } # End gatherOneFS ### # gatherOneFSVersions: Gathers OneFSVersion from all nodes. sub gatherOneFSVersions { return if wasCalled(); gatherUname(); gatherOneFS(); gatherArrayXML(); # Assign variables foreach my $lnn (@LNNS) { # Parse uname for a OneFS version my $version_uname = ""; if ($NODE_INFO{$lnn}{'uname'} and $NODE_INFO{$lnn}{'uname'} =~ /[ v](\d+\.\d+\.\d+\.\d+) /) { $version_uname = $1; $NODE_INFO{$lnn}{'OneFS'} = $1; } # In order of reliability for gathering the OneFS version: # array.xml, node_status.gc, and uname output my $version = ""; if ($NODE_INFO{$lnn}{'onefs_version'}) { $version = lc $NODE_INFO{$lnn}{'onefs_version'}; } elsif ($NODE_INFO{$lnn}{'current_onefs_version'}) { $version = lc $NODE_INFO{$lnn}{'current_onefs_version'}; } elsif ($NODE_INFO{$lnn}{'uname'} and $NODE_INFO{$lnn}{'uname'} =~ /(0x[0-9a-f]+)/i) { $version = lc $1; } if ($version) { my %version = %{parseOneFSVersion($version)}; if (!$NODE_INFO{$lnn}{'OneFS'}) { $NODE_INFO{$lnn}{'OneFS'} = sprintf("%s.%s.%s.%s", $version{'major'}, $version{'minor'}, $version{'sub'}, $version{'dot'}); } if (!$CLUSTER_INFO{'onefs_version'} or $CLUSTER_INFO{'onefs_version'} lt $version) { $CLUSTER_INFO{'onefs_version'} = $version; $CLUSTER_INFO{'Build'} = $version{'build'}; $CLUSTER_INFO{'Style'} = $version{'style'}; } } if (!$NODE_INFO{$lnn}{'OneFS'}) { next } debug("NODE_INFO->${lnn}->OneFS=$NODE_INFO{$lnn}{'OneFS'}"); if (!isCurrentOneFS(">$NODE_INFO{$lnn}{'OneFS'}")) { $CLUSTER_INFO{'OneFS'} = $NODE_INFO{$lnn}{'OneFS'}; } } debug("CLUSTER_INFO->onefs_version=$CLUSTER_INFO{'onefs_version'}") if $CLUSTER_INFO{'onefs_version'}; debug("CLUSTER_INFO->OneFS=$CLUSTER_INFO{'OneFS'}") if $CLUSTER_INFO{'OneFS'}; debug("CLUSTER_INFO->Build=$CLUSTER_INFO{'Build'}") if $CLUSTER_INFO{'Build'}; debug("CLUSTER_INFO->Style=$CLUSTER_INFO{'Style'}") if $CLUSTER_INFO{'Style'}; if ($CLUSTER_INFO{'Style'} and $CLUSTER_INFO{'Style'} eq 0x3) { $CLUSTER_INFO{'Beta'} = 1; debug("CLUSTER_INFO->Beta=1"); } # Abort if unable to determine what OneFS version is currently in use if (!$CLUSTER_INFO{'OneFS'}) { fail("Unable to determine a OneFS version, checks will not proceed."); printCheck("Gathering OneFS Versions"); printResults(); exit 1; } } # End gatherOneFSVersions ### # gatherUname: Gathers the uname output from each node sub gatherUname { return if wasCalled(); # Gather information my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array uname -a]); } else { $gather = gatherCluster(qq[2>&1 grep -H . */uname]); } # Assign variables foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; foreach my $line (@{$gather->{$lnn}}) { $NODE_INFO{$lnn}{'uname'} = $line; debug("NODE_INFO->${lnn}->uname=$line"); } } } # End gatherUname ### # generateTargetVersions: Generates an array of OneFS versions within # a specific family that are greater than or equal to the provided # version sub generateTargetVersions { my ($version) = @_; if (!$version) { return } debug("generateTargetVersions: Argument: $version"); our %Code_Matrix, %Patch_Releases; my @destinations = merge(keys %Code_Matrix, keys %Patch_Releases); @destinations = grep {$_ =~ /^\d+\.\d+\.\d+\.\d+$/} @destinations; @destinations = sort {compareVersions($b, $a)} @destinations; my ($major, $minor, $service) = split(/\./, $version); my @version_array = (); foreach (@destinations) { if (compareVersions($_, "$major.$minor.$service") == 0 and compareVersions($_, $version) >= 0) { push(@version_array, $_); } } debug("generateTargetVersions: Return: [ @version_array ]"); return @version_array; } # End generateTargetVersions ### # generateUpgradePaths: Generates an array for the upgrade path sub generateUpgradePaths { my ($source, $target) = @_; if (!$source or !$target) { debug('Insufficient arguments provided'); return (); } debug("Source: $source, Destination: $target"); our %Code_Matrix; # The first call to generateUpgradePaths if (isDestinationOneFS($target) and isCurrentOneFS($source)) { # Force OneFS 7.1.1.11 as an interim jump when upgrading to OneFS 8.1.1 # or later and upgrading from OneFS 7.1.1.10 or earlier if (isDestinationOneFS('>=8.1.1') and isCurrentOneFS('<7.1.1.11')) { my @path_to_interim = generateUpgradePaths(isCurrentOneFS(), '7.1.1.11'); my @path_from_interim = generateUpgradePaths('7.1.1.11', isDestinationOneFS()); # If no upgrade path is found, return if (!@path_to_interim or !@path_from_interim) { return () } # Return the combined path pop(@path_to_interim); return @path_to_interim, @path_from_interim; } # Force OneFS 8.2.2.0 as an interim jump if at risk for KB 546604 if ($RPS{'KB546604'} and isDestinationOneFS('>=9') and isCurrentOneFS('<=8.1')) { my @path_to_interim = generateUpgradePaths(isCurrentOneFS(), '8.2.2.0'); my @path_from_interim = generateUpgradePaths('8.2.2.0', isDestinationOneFS()); # If no upgrade path is found, return if (!@path_to_interim or !@path_from_interim) { return () } # Return the combined path pop(@path_to_interim); return @path_to_interim, @path_from_interim; } # Allow upgrade to 8.2.0 build 11 from 8.2.0 build 9 if (isCurrentOneFS('8.2.0.0') and isDestinationOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11) { return ('8.2.0.0', '8.2.0.0'); } # Allow upgrade to 8.2.0 build 4 from 8.2.0 build 3 if (isCurrentOneFS('8.2.1.0') and isDestinationOneFS('8.2.1.0') and $CLUSTER_INFO{'Build'} < 4) { return ('8.2.1.0', '8.2.1.0'); } } # Recursion call where multiple jumps are needed if (!isCurrentOneFS($source)) { # This may need revision later # Avoid OneFS 9.2 as an interim jump due to post-upgrade data restriping if ($source =~ /^9\.2./) { return () } # Avoid OneFS 7.1.1.7/8 as an interim jump when going to 7.2.1.3 if ($target eq '7.2.1.3' and $source =~ /^7\.1\.1\.[89]$/) { return (); } # Avoid OneFS 8.0.1.0 as an interim jump, we don't like 8.0.1.0 if ($source eq '8.0.1.0') { return () } # Avoid OneFS 8.1.3.0 as an interim jump as it is DA if ($source eq '8.1.3.0') { return () } # Avoid OneFS 8.0 as an interim jump when going from less than # OneFS 7.1.1 to OneFS 8.1.0 or higher if (compareVersions($target, '8.1') >= 0 and isCurrentOneFS('<=7.1.1') and compareVersions($source, '8.0') == 0) { return (); } # Avoid OneFS 7.2.0 as an interim jump when going from less than 7.1.1 # to 7.2.1 or higher if (compareVersions($target, '7.2.1') >= 0 and isCurrentOneFS('<=7.1.1') and compareVersions($source, '7.2.0') == 0) { return (); } } # Locate the key in the code matrix to use for out-of-family upgrades my ($key) = grep {compareVersions($source, $_) == 0} keys %Code_Matrix; debug("Source key: $key"); # Generate destination versions my @destinations = (); foreach my $version (@{$Code_Matrix{$key}}, $source) { foreach (generateTargetVersions($version)) { if (compareVersions($_, $source) > 0 and compareVersions($_, $target) <= 0) { push(@destinations, $_); } } } # Sort the destination versions @destinations = sort {compareVersions($b, $a)} @destinations; debug("Destination version array: @destinations"); # Search for the destination version and return if found my ($results) = grep {compareVersions($_, $target) == 0} @destinations; if ($results) { return ($source, $target) } # If the destination was not found, it may be a multi-jump upgrade debug('No match found, searching for multiple jumps'); foreach (@destinations) { my @upgrade_path = generateUpgradePaths($_, $target); if (@upgrade_path) { return ($source, @upgrade_path) } } # No paths found, return return (); } # End generateUpgradePaths # checkKB520799: Checks for criteria of KB 520799 sub checkKB520799 { if (wasCalled()) { return } if (!isLive()) { return } if (!hasPatch('patch-208294')) { return } # Setup my $CMD = qq[2>&1 ${SUDO} isi_for_array 'ls /lib/libcrypto.so*']; my $RE_PATCHED = qr/\/lib\/libcrypto\.so\.8\.patched/; my $SPF_MISSING = 'patch-208294 is installed but'. ' /lib/libcrypto.so.8.patched is missing on the following nodes: %s.'. ' Follow resolution steps in KB 520799'. ' [https://www.dell.com/support/kbdoc/000036454 - Patch 208294 is'. ' missing /lib/libcrypto.so.8.patched]'; # Gather my $gather = gatherCluster($CMD); # Analyze my @nodes_missing_patched = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } my ($res) = grep {/$RE_PATCHED/} @{$gather->{$lnn}}; if (!$res) { push(@nodes_missing_patched, $lnn) } } # Flag if (@nodes_missing_patched) { fail(sprintf($SPF_MISSING, compressRange(@nodes_missing_patched))); } } # End checkKB520799 sub checkPatch146974 { local $patch = "patch-146974"; local $re_versions = qr/^(7\.1\.1\.2)$/; local $impacts = "During installation or removal of $patch, the SMB and authentication services are unavailable and SMB client connections to the cluster are disconnected. After the patch is installed, SMB clients must reestablish a connection to the cluster."; checkPatch(); } sub checkPatch147686 { local $patch = "patch-147686"; local $re_versions = qr/^(7\.1\.0\.(4|5|6))$/; local $impacts = "During installation or removal of $patch, the SMB and authentication services are unavailable and SMB client connections to the cluster are disconnected. After the patch is installed, SMB clients must reestablish a connection to the cluster."; checkPatch(); } sub checkPatch161166 { local $patch = "patch-161166"; local $re_versions = qr/^(7\.2\.0\.2)$/; local $impacts = "During installation or removal of $patch, the SMB and authentication services are unavailable and SMB client connections to the cluster are disconnected. After the patch is installed, SMB clients must reestablish a connection to the cluster."; checkPatch(); } sub checkPatch164666 { local $patch = "patch-164666"; local $re_versions = qr/^(7\.1\.1\.(0|1|2|3|4|5|6|7|8)|7\.2\.1\.(0|1|2))$/; checkPatch(); } sub checkPatch175451 { gatherNetworkInterfaces(); local $patch = "patch-175451"; local $re_versions = qr/^(7\.2\.0\.(0|1|2|3|4|5))$/; local $impacts = "During installation or removal of patch 175451, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch178793 { local $patch = "patch-178793"; local $re_versions = qr/^(8\.0\.0\.(0|1))$/; local $impacts = "During installation or removal of $patch, the isi_papi_d process and SmartConnect service are stopped. As a result, CLI commands beginning with the 'isi' prefix cannot be run and both the OneFS web administration interface and the SmartConnect service are unavailable. After installation or removal of this patch, the isi_papi_d process and SmartConnect service restart automatically."; checkPatch(); } sub checkPatch179385 { local $patch = "patch-179385"; local $re_versions = qr/^(8\.0\.0\.2)$/; local $impacts = "During installation or removal of $patch, the isi_papi_d process and SmartConnect service are stopped. As a result, CLI commands beginning with the 'isi' prefix cannot be run and both the OneFS web administration interface and the SmartConnect service are unavailable. After installation or removal of this patch, the isi_papi_d process and SmartConnect service restart automatically."; checkPatch(); } sub checkPatch181490 { local $patch = "patch-181490"; local $re_versions = qr/^(8\.0\.1\.0)$/; local $impacts = "During installation or removal of $patch, the isi_papi_d process and SmartConnect service are stopped. As a result, CLI commands beginning with the 'isi' prefix cannot be run and both the OneFS web administration interface and the SmartConnect service are unavailable. After installation or removal of $patch, the isi_papi_d process and SmartConnect service restart automatically."; checkPatch(); } sub checkPatch188239 { gatherNetworkInterfaces(); local $patch = "patch-188239"; local $re_versions = qr/^(8\.0\.0\.[01234])$/; local $impacts = "During installation or removal of patch 188239, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch188660 { local $patch = "patch-188660"; local $re_versions = qr/^(8\.0\.1\.0)$/; local @deprecated_patches = hasPatch("patch-(187743|186696|185821|184615)"); local $impacts = "During installation or removal of $patch, the service that manages SMB, NFS, HDFS, and Swift client connections as well as authentication to the cluster is restarted. As a result, during installation or removal of $patch, client connections to the cluster over these protocols might be disrupted and new client connections cannot be authenticated. FTP and HTTP connections might be affected in some environments, and client connections might have to be reestablished manually after the installation or removal process is complete."; checkPatch(); } sub checkPatch189132 { gatherNetworkInterfaces(); local $patch = "patch-189132"; local $re_versions = qr/^(7\.1\.1\.(0|1|2|3|4|5|6|7|8|9|10|11))$/; local $impacts = "During installation or removal of patch 189132, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces might be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch189133 { gatherNetworkInterfaces(); local $patch = "patch-189133"; local $re_versions = qr/^(7\.2\.1\.[012345])$/; local @deprecated_patches = hasPatch("patch-(175451)"); local $impacts = "During installation or removal of patch 189133, clients and applications that are connected to the cluster through BXE 10 GigE network interfaces may be disconnected. These connections must be reestablished after the patch installation is complete." if $CLUSTER_INFO{'BXE Nodes'}; local @reboot = @{$CLUSTER_INFO{'BXE Nodes'}} if $CLUSTER_INFO{'BXE Nodes'}; checkPatch(); } sub checkPatch206322 { local $patch = "patch-206322"; local $re_versions = qr/^(8\.0\.0\.4)$/; local @deprecated_patches = hasPatch("patch-(187902|188699|189708|189853|191603|194238|194268|195171|196308|196448|196845|196906|197503|197648|198834|199265)"); local @conflicting_patches = hasPatch("patch-(191606|194795|196968|197390|197710|197902|199123|200631|200950|200951|203704|203895)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: SMB, Likewise, NFS, HDFS, Isi Migrate, CloudPools, Flexnet, SmartConnect, NDMP, PAPI, Tardis"; # Patch 206322 is deprecated by patch 212411 if (!isDestinationOneFS() and isCurrentOneFS($re_versions) and hasPatch('patch-212411')) { return; } checkPatch(); } sub checkPatch213286 { local $patch = "patch-213286"; local $re_versions = qr/^(7\.2\.1\.5)$/; local @deprecated_patches = hasPatch("patch-(211400)"); local @conflicting_patches = hasPatch("patch-(200764|205288)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: webui, apache2, sshd"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; checkPatch(); } sub checkPatch213287 { local $patch = "patch-213287"; local $re_versions = qr/^(7\.2\.1\.6)$/; local @conflicting_patches = hasPatch("patch-(200764|205288)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: webui, apache2, sshd"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; checkPatch(); } sub checkPatch219279 { local $patch = "patch-219279"; local $re_versions = qr/^(8\.0\.1\.2)$/; local @deprecated_patches = hasPatch("patch-(216929|216763|215881)"); local @conflicting_patches = hasPatch("patch-(215020|213630)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: SMB, Likewise, NFS, HDFS, SNMP, isi_avscan_d, Migrate, pcnfsd, NDMP, PAPI, Tardis"; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch220098 { local $patch = "patch-220098"; local $re_versions = qr/^(8\.1\.0\.2)$/; local @deprecated_patches = hasPatch("patch-(219689|224219|227398)"); local @conflicting_patches = hasPatch("patch-(222005)"); local $impacts = "During installation or removal of $patch, the following services are restarted: auth, celog, hdfs, isi_hwmon, isi_smartconnect, isi_stats_d, isi_stats_hist_d, lwdm, lwswift, nfs, smb"; local $service_restart = 1; checkPatch(); } sub checkPatch220176 { local $patch = "patch-220176"; local $re_versions = qr/^(8\.0\.1\.2)$/; local @deprecated_patches = hasPatch("patch-(213280)"); local $impacts = "During installation or removal of $patch, the following services are restarted: isi_mcp, apache2, isi_webui"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; checkPatch(); } sub checkPatch221602 { local $patch = "patch-221602"; local $re_versions = qr/^(8\.0\.0\.4)$/; local @conflicting_patches = hasPatch("patch-(208294|205289)"); local $important = "IMPORTANT: If you update Isilon OneFS with patch 221602, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing patch 221602 due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch221603 { local $patch = "patch-221603"; local $re_versions = qr/^(8\.0\.0\.5)$/; local @deprecated_patches = hasPatch("patch-(217637|211167|209258|208294|207787|205289)"); local $impacts = "During installation or removal of $patch, the following services are restarted: mcp, webui, http, isi_papi_d, isi_object_d"; local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\n IMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch221947 { local $patch = "patch-221947"; local $re_versions = qr/^(8\.1\.0\.1)$/; local @deprecated_patches = hasPatch("patch-(213282|217638)"); local @conflicting_patches = hasPatch("patch-(211962|220053|194276)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch221949 { local $patch = "patch-221949"; local $re_versions = qr/^(8\.1\.0\.0)$/; local @deprecated_patches = hasPatch("patch-(213281)"); local @conflicting_patches = hasPatch("patch-(211967|211961|205291|200762)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch222487 { local $patch = "patch-222487"; local $re_versions = qr/^(8\.0\.1\.1)$/; local @deprecated_patches = hasPatch("patch-(213279|211960)"); local @conflicting_patches = hasPatch("patch-(215497|211403|210691|207276|205290)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing $patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch223311 { local $patch = "patch-223311"; local $re_versions = qr/^(8\.0\.0\.5)$/; local @deprecated_patches = hasPatch("patch-(211169|217838)"); local @conflicting_patches = hasPatch("patch-(203466|204429|204812|204900|206113|210031|210400|211170|211795|213989|215592|215685|215703|216731)"); local $impacts = "During installation or removal of $patch, the following services are restarted: NFS, PC NFS Server, SMB, LIKEWISE, HDFS, CloudPools, Tardis, PAPI, SmartConnect, NDMP, Authentication, LWSM, Swift, SNMP, FlexNet, JobEngine, Statistics & Monitoring, Statistics History"; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch223338 { local $patch = "patch-223338"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(208154|214178|215884|219375|219904|221633|224127|224804|227485|231811)"); local @conflicting_patches = hasPatch("patch-(222935|218751)"); local $impacts = "During installation or removal of $patch, the following services are restarted: auth, celog, hdfs, isi_avscan_d, isi_cpool_d, isi_cpool_io_d, isi_flexnet_d, isi_hangdump, isi_job_d, isi_km_d, isi_mcp, isi_papi_d, isi_smartconnect, isi_stats_d, isi_stats_hist_d, lwsm, lwswift, nfs, smb, snmp"; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch224116 { local $patch = "patch-224116"; local $re_versions = qr/^(8\.1\.1\.1)$/; local @deprecated_patches = hasPatch("patch-(222493)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch226323 { local $patch = "patch-226323"; local $re_versions = qr/^(8\.1\.0\.2)$/; local @deprecated_patches = hasPatch("patch-(220928|222464|223838)"); local @conflicting_patches = hasPatch("patch-(223989)"); local $service_restart = 1; local $priority = 10; local @reboot = @LNNS; checkPatch(); } sub checkPatch226984 { local $patch = "patch-226984"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(225079)"); local $service_restart = 1; local $priority = 10; local @reboot = @LNNS; checkPatch(); } sub checkPatch228458 { local $patch = "patch-228458"; local $re_versions = qr/^(8\.1\.0\.2)$/; local @deprecated_patches = hasPatch("patch-(213283|219709|221946|223678)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch229156 { local $patch = "patch-229156"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(215690|221638)"); local $impacts = "During installation or removal of $patch, the following services are restarted: SyncIQ, NDMP, PAPI, MCP"; local $service_restart = 1; checkPatch(); } sub checkPatch229158 { local $patch = "patch-229158"; local $re_versions = qr/^(8\.0\.0\.5)$/; local @deprecated_patches = hasPatch("patch-(203734|211168|203743|213179|217463|224399)"); local @conflicting_patches = hasPatch("patch-(206113|210400|211169|217838)"); local $impacts = "During installation or removal of $patch, the following services are restarted: SyncIQ, PAPI"; local $service_restart = 1; checkPatch(); } sub checkPatch229160 { local $patch = "patch-229160"; local $re_versions = qr/^(8\.1\.0\.2)$/; local $impacts = "During installation or removal of $patch, the following services are restarted: SyncIQ, PAPI, NDMP"; local $service_restart = 1; local @deprecated_patches = hasPatch("patch-(222718)"); checkPatch(); } sub checkPatch231386 { local $patch = "patch-231386"; local $re_versions = qr/^(8\.0\.0\.6)$/; local @deprecated_patches = hasPatch("patch-(213278|221617|223243)"); local @conflicting_patches = hasPatch("patch-(228563)"); local $important = "IMPORTANT: If you update Isilon OneFS with $patch, and you are using Insight IQ, you must upgrade to Insight IQ 4.1.2 prior to installing the patch.\nIMPORTANT: For complete protection against these vulnerabilities, you must disable basic authentication. Additionally, if you are installing $patch due to cross-site request forgery (CSRF) concerns, AND you have custom OneFS API-based automation, you must also update your automation to incorporate anti-CSRF tokens provided by your Isilon cluster. For more information about these two procedures, see https://www.dell.com/support/kbdoc/517421 on the Customer support website."; local $service_restart = 1; local @reboot = @LNNS; checkPatch(); } sub checkPatch233663 { local $patch = "patch-233663"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @deprecated_patches = hasPatch("patch-(232890|234237)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: celog, isi_cpool_d, isi_cpool_io_d, isi_drive_repurpose_d, isi_job_d, isi_migrate, isi_ph_rpcd, isi_smartpools_d, ndmpd, isi_papi_d, auth, hdfs, isi_mcp, isi_tardis_gcfg_d, isi_tardis_d, lwsm, lwswift, nfs, smb"; local $important = "The OneFS Protocols/Userspace Rollup patch, $patch, should be considered mandatory for every upgrade to OneFS 8.0.0.7."; local $service_restart = 1; local $priority = 10; checkPatch(); } sub checkPatch8007HC { local $patch = 'HealthCheck_8.0.0.7_2020-02'; local $re_versions = qr/^8\.0\.0\.7$/; local @deprecated_patches = hasPatch('^HealthCheck_8.0.0.7_(?:v|2019|2020-01)'); local $impacts = "During installation or removal of $patch, the platform API service (isi_papi_d) will be restarted along with the healthcheck service."; local $important = "$patch requires that patch-240796 is first installed"; local @requires = ('patch-240796'); checkPatch(); } sub checkPatch8007Kernel { local $patch = "patch-240795"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @conflicting_patches = hasPatch("patch-(233621)"); local $service_restart = 1; local @reboot = @LNNS; local $priority = 5; checkPatch(); } sub checkPatch8007Userspace { local $patch = "patch-240796"; local $re_versions = qr/^(8\.0\.0\.7)$/; local @deprecated_patches = hasPatch("patch-(232890|233663|234237|236720|237180|240436|240514|241945)"); local @conflicting_patches = hasPatch("HealthCheck_8.0.0.7_(v1.0|v2.0)|patch-(227974|242012|243995)"); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: apache2, auth, celog, hdfs, isi_audit_cee, isi_audit_d, isi_audit_syslog, isi_cpool_d, isi_cpool_io_d, isi_drive_d, isi_drive_repurpose_d, isi_drivemon, isi_hangdump, isi_healthcheck, isi_job_d, isi_km_d, isi_mcp, isi_migrate, isi_papi_d, isi_ph_rpcd, isi_object_d, isi_smartpools_d, isi_smartquotas, isi_tardis_d, isi_tardis_gcfg_d, isi_webui_d, lwsm, lwswift, ndmpd, nfs, rpcbind, smb"; local $service_restart = 1; local $priority = 10; checkPatch(); } sub checkPatch8104Kernel { local $patch = "8.1.0.4_KGA-RUP_2019-09_256245"; local $re_versions = qr/^(8\.1\.0\.4)$/; local @conflicting_patches = hasPatch("patch-(235853|241067)"); local @deprecated_patches = hasPatch('^(8\.1\.0\.4|patch).*(228202|236488|236994|240815|242570|243892|244713|245399|246716|248573|249719|250863|252460|254526)$'); local @reboot = @LNNS; local $priority = 5; local $important = " Patch $patch includes a number of changes that may affect existing configurations: * ssh(1)/sshd(8): Drop compatibility support for some very old SSH implementations, including ssh.com <=2. and OpenSSH <= 3.. These versions were all released in or before 2001 and predate the final SSH RFCs. The support in question isn't necessary for RFC-compliant SSH implementations. * ssh(1): Delete SSH protocol version 1 support, associated configuration options and documentation. * ssh(1)/sshd(8): Remove support for the hmac-ripemd160 MAC. * ssh(1)/sshd(8): Remove support for the arcfour, blowfish and CAST ciphers. * Refuse RSA keys <1024 bits in length and improve reporting for keys that do not meet this requirement. * ssh(1): Do not offer CBC ciphers by default. * This patch deprecates the sshd_config UsePrivilegeSeparation option, thereby making privilege separation mandatory. Privilege separation has been on by default for almost 15 years and sandboxing has been on by default for almost the last five. "; checkPatch(); } sub checkPatch8104Userspace { local $patches_with_svc_scripts = qr/patch-(222280|228133|231017|231712|231786|232742|233356|234345|234538|235862|236576|237861|239173|239649|240437|240815|241584|241946|242136|243890|243891|245397|245398|246957|247138|247848|249239)/; # GA RUP details local $patch = "8.1.0.4_UGA-RUP_2019-09_256243"; local $re_versions = qr/^(8\.1\.0\.4)$/; local @conflicting_patches = hasPatch('^(8\.1\.0\.4|patch).*(228133|231712|232711|234538|235853|241067|242136|243891|245398|246667|246715|246957|247138|248572|248825|249239|249289|249718|250862|251180|252458|252459|254525|256244)$'); local @deprecated_patches = hasPatch('^(8\.1\.0\.4|patch).*(222280|231017|233356|234345|239173|239649|241096|241584|241946|243890|245397|246714|247848|248571|249717|250861|254524)$'); local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: apache2, auth, celog, connectemc, dmilog, hdfs, ifsd, isi_audit_cee, isi_audit_d, isi_audit_syslog, isi_avscan_d, isi_boot_d, isi_cbind_d, isi_cpool_d, isi_cpool_io_d, isi_drive_d, isi_drive_repurpose_d, isi_drivemon, isi_esrs_d, isi_flexnet_d, isi_group_change_d, isi_hangdump, isi_hwmon, isi_job_d, isi_join_d, isi_km_d, isi_mcp, isi_migrate, isi_object_d, isi_papi_d, isi_ph_rpcd, isi_protohealth_d, isi_replicate, isi_rpc_d, isi_rsapi_d, isi_sasphymon, isi_smartconnect, isi_smartpools_d, isi_smartquotas, isi_snapshot_d, isi_stats_d, isi_stats_hist_d, isi_tardis_d, isi_tardis_gcfg_d, isi_upgrade_d, isi_vasa_d, isi_webui, lwsm, lwswift, ndmpd, nfs, pcnfsd, rpcbind, scheduler, smb, snmp, syslogd"; local $service_restart = 2; local $priority = 10; # DA RUP details if (hasPatch('^(8\.1\.0\.4|patch).*(228133|231712|231786|232742|234538|235862|236576|237861|242136|243891|245398|246715|246957|247138|248572|249239|249718|250862|252458|252459|254525|256244)$')) { $patch = "8.1.0.4_UDA-RUP_2019-09_256244"; @conflicting_patches = hasPatch('^(8\.1\.0\.4|patch).*(232711|235853|240437|241067|246667|248825|251180)$'); @deprecated_patches = hasPatch('^(8\.1\.0\.4|patch).*(222280|228133|231017|231712|231786|232742|233356|234345|234538|235862|236576|237861|239173|239649|241096|241584|241946|242136|243890|243891|245397|245398|246714|246715|246957|247138|247848|248571|248572|249239|249717|249718|250861|250862|252458|252459|254524|254525)$'); $impacts = "During the installation of removal of $patch, the following services are restarted: apache2, auth, celog, connectemc, cron, dmilog, gmond, hdfs, ifsd, isi_audit_cee, isi_audit_d, isi_audit_syslog, isi_avscan_d, isi_boot_d, isi_cbind_d, isi_cpool_d, isi_cpool_io_d, isi_drive_d, isi_drive_repurpose_d, isi_drivemon, isi_esrs_d, isi_flexnet_d, isi_group_change_d, isi_hangdump, isi_hwmon, isi_ipmi_d, isi_job_d, isi_join_d, isi_km_d, isi_mcp, isi_migrate, isi_object_d, isi_papi_d, isi_ph_rpcd, isi_protohealth_d, isi_replicate, isi_rpc_d, isi_rsapi_d, isi_sasphymon, isi_smartconnect, isi_smartpools_d, isi_smartquotas, isi_snapshot_d, isi_stats_d, isi_stats_hist_d, isi_tardis_d, isi_tardis_gcfg_d, isi_upgrade_d, isi_vasa_d, isi_webui, lwsm, lwswift, ndmpd, nfs, ntpd, pcnfsd, rpcbind, scheduler, smb, snmp, syslogd, xinetd"; } checkPatch(); } sub checkPatch810HC { local $patch = 'HealthCheck_8.1.0_2021-05'; local $re_versions = qr/^8\.1\.0\.4$/; local @deprecated_patches = hasPatch('^HealthCheck_8.1.0_(?:v|2019|2020|2021-0[123])'); checkPatch(); } sub checkPatch8120DA { if (!hasPatch('^(?:patch|8\.1\.2).*[-_](?:43|103|221|367|405|482|525|644|727|825|864|998|1102|1195|239260|240798|243624|243896|245049|245401|245889|245974|246059|246712|248569|248876|249618|249621|249928|250807|251712|252456|254384|255591|256177|257611|257847|258683|259126|261109|262896|264011|264785|266228|269474|269974|270946|272855)$')) { return } local $patch = '8.1.2.0_DA-RUP_2021-06_PSP-1195'; local $re_versions = qr/^8\.1\.2\.0$/; local @reboot = @LNNS; local $priority = 5; local @requires = ('8.1.2.0_UGA-PATCH-INFRA_2019-09_255624'); # @deprecated left for use with patch infrastructure patch conflicts local @deprecated_patches = hasPatch('^(?:patch|8\.1\.2).*[-_](?:240794|243897|245402|246713|248570|249622|250808|252457|254360|256016)$'); local @replaceable_patches = hasPatch('^(?:patch|8\.1\.2).*[-_](?:42|43|102|103|193|220|221|265|366|367|405|479|482|525|643|644|726|727|825|826|863|864|997|998|1101|1102|1194|236288|237113|237483|239260|239343|240163|240798|241940|241947|242854|243624|243894|243896|243912|245049|245400|245401|245889|245974|246059|246545|246711|246712|248568|248569|248876|249618|249620|249621|249770|249928|250806|250807|250901|251188|251712|251937|252455|252456|252507|254383|254384|254385|255591|256176|256177|256178|257610|257611|257612|257847|258052|258683|259125|259126|259127|260832|261108|261109|261110|262172|262895|262896|262897|264011|264566|264784|264785|264786|265016|265068|266227|266228|266229|266931|269352|269353|269473|269474|269475|269847|269974|270944|270946|272854|272855)$'); local @conflicting_patches = hasPatch('^(?:patch|8\.1\.2).*[-_](?:222|241673|246754|249151|250131|251091|252360|262667|263651|268122|268621|269607|271537)$'); checkPatch(); } sub checkPatch8120GA { if (hasPatch('^(?:patch|8\.1\.2).*[-_](?:43|103|221|367|405|482|525|644|727|825|864|998|1102|1195|239260|240798|243624|243896|245049|245401|245889|245974|246059|246712|248569|248876|249618|249621|249928|250807|251712|252456|254384|255591|256177|257611|257847|258683|259126|261109|262896|264011|264785|266228|269474|269974|270946|272855)$')) { return } local $patch = '8.1.2.0_GA-RUP_2021-06_PSP-1194'; local $re_versions = qr/^8\.1\.2\.0$/; local @reboot = @LNNS; local $priority = 5; local @requires = ('8.1.2.0_UGA-PATCH-INFRA_2019-09_255624'); # @deprecated left for use with patch infrastructure patch conflicts local @deprecated_patches = hasPatch('^(?:patch|8\.1\.2).*[-_](?:240794|243897|245402|246713|248570|249622|250808|252457|254360|256016)$'); local @replaceable_patches = hasPatch('^(?:patch|8\.1\.2).*[-_](?:42|102|193|220|265|366|479|643|726|826|863|997|1101|236288|237113|237483|239343|240163|241940|241947|242854|243894|243912|245400|246545|246711|248568|249620|249770|250806|250901|251188|251937|252455|252507|254383|254385|256176|256178|257610|257612|258052|259125|259127|260832|261108|261110|262172|262895|262897|264566|264784|264786|265016|265068|266227|266229|266931|269352|269353|269473|269475|269847|270944|272854)$'); local @conflicting_patches = hasPatch('^(?:patch|8\.1\.2).*[-_](?:43|103|221|222|367|405|482|525|644|727|825|864|998|1102|1195|239260|240798|241673|243624|243896|245049|245401|245889|246059|246712|246754|248569|248876|249151|249618|249621|249928|250131|250807|251091|251712|252360|252456|254384|255591|256177|257611|257847|258683|259126|261109|262667|262896|263651|264011|264785|266228|268122|268621|269474|269607|269974|270946|271537|272855)$'); checkPatch(); } sub checkPatch8120Patch { local $patch = '8.1.2.0_UGA-PATCH-INFRA_2019-09_255624'; local $re_versions = qr/^(8\.1\.2\.0)/; local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: isi_papi_d"; local $priority = 15; local @requires = (); my @conflicts = hasPatch('^(?:patch|8\.1\.2\.0).*(?:240794|243897|245402|246713|248570|248876|249622|250808|252457|252639|254360|256016)$'); foreach my $p (@conflicts) { push(@requires, "uninstall $p"); } if (!hasPatch('^8.1.2.0_UGA-PATCH-INFRA_')) { checkPatch() } $patch = '8.1.2.0_UGA-PATCH-INFRA_2020-08_PSP-268'; $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: isi_papi_d"; @requires = ('8.1.2.0_UGA-PATCH-INFRA_2019-09_255624'); checkPatch(); } sub checkPatch812HC { local $patch = 'HealthCheck_8.1.2_2021-05'; local $re_versions = qr/^8\.1\.2\.0$/; local @requires = ('8.1.2.0_UGA-PATCH-INFRA_2019-09_255624'); local @replaceable_patches = hasPatch('^HealthCheck_8.1.2_(?:v|2019|2020|2021-0[123])'); checkPatch(); } sub checkPatch813HC { local $patch = 'HealthCheck_8.1.3_2021-05'; local $re_versions = qr/^8\.1\.3\.0$/; local @deprecated_patches = hasPatch('^HealthCheck_8.1.3_(?:2019|2020|2021-0[123])'); checkPatch(); } sub checkPatch813Kernel { local $patch = "patch-244941"; local $re_versions = qr/^(8\.1\.3\.0)$/; local @reboot = @LNNS; local $service_restart = 1; local $priority = 5; checkPatch(); } sub checkPatch813Userspace { local $patch = "patch-244942"; local $re_versions = qr/^(8\.1\.3\.0)$/; local $impacts = "During installation or removal of $patch, the following services are restarted: apache2, auth, celog, hdfs, httpd, isi_audit_cee, isi_audit_d, isi_audit_syslog, isi_cpool_d, isi_cpool_io_d, isi_drive_repurpose_d, isi_drivemon, isi_group_change_d, isi_hangdump, isi_hwmon, isi_km_d, isi_mcp, isi_migrate, isi_object_d, isi_papi_d, isi_ph_rpcd, isi_rsapi_d, isi_smartpools_d, isi_tardis_d, isi_webui, lwsm, lwswift, ndmpd, nfs, smb"; local @reboot = @LNNS; local $service_restart = 1; local $priority = 10; checkPatch(); } sub checkPatch820GA { return if not isDestinationOneFS('8.2.0.0') and isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11; if (hasPatch('^8\.2\.0.*(?:250838|254106|254793|254798|255568|257480|257594|259123|261104|262893|264782)')) { return } local $patch = '8.2.0_GA-RUP_2020-06_273332'; local $re_versions = qr/^8\.2\.0\.0$/; local @requires = ('8.2.0_UGA-PATCH-INFRA_2019-08_251917'); local @replaceable_patches = hasPatch('^8\.2\.0.*(?:250837|250839|254105|254107|254527|254792|254807|254894|255567|255569|256081|256808|256855|256885|257593|257595|257853|258529|258860|259122|259124|259916|260818|261103|261105|262892|262894|264781|264783|267633|268444|269057|269435|269642)$'); local @conflicting_patches = hasPatch('^8\.2\.0.*(?:250838|252451|252624|254106|254695|254793|254798|255568|257480|257594|259123|260301|260849|261104|262893|264647|264782|264803|266617|266990|270126|271477|271507)$'); local $priority = 5; local @reboot = @LNNS; checkPatch(); } sub checkPatch820HC { local $patch = 'HealthCheck_8.2.0_2021-09'; local $re_versions = qr/^8\.2\.0\.0$/; local @requires = ('8.2.0_UGA-PATCH-INFRA_2019-08_251917'); local @replaceable_patches = hasPatch('^HealthCheck_8.2.0_(?:2019|2020|2021-0[1235])'); checkPatch(); } sub checkPatch820Kernel { return if not isDestinationOneFS('8.2.0.0') and isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11; if (!hasPatch('^8\.2\.0.*(?:250838|254106|254793|254798|255568|257480|257594|259123|261104|262893|264782)')) { return } local $patch = '8.2.0_KGA-RUP_2020-03_270126'; local $re_versions = qr/^8\.2\.0\.0$/; local @requires = ('8.2.0_UGA-PATCH-INFRA_2019-08_251917'); local @replaceable_patches = hasPatch('^8\.2\.0.*(?:250839|254107|254527|255569|256808|257595|258529|258860|259124|260818|261105|262894|264783)'); local @conflicting_patches = hasPatch('^8\.2\.0.*(?:252451|252624|254695|264803|267633|268444|269057)'); local $priority = 5; local @reboot = @LNNS; checkPatch(); } sub checkPatch820Patch { if (hasPatch('8.2.0_UGA-PATCH-INFRA_2019-11_263084')) { return } local $patch = '8.2.0_UGA-PATCH-INFRA_2019-08_251917'; local $re_versions = qr/^(8\.2\.0)/; local $impacts = "During installation or removal of $patch, the following services will be stopped and restarted automatically: isi_papi_d"; local $priority = 15; return if not isDestinationOneFS('8.2.0.0') and isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11; checkPatch(); } sub checkPatch820Userspace { return if not isDestinationOneFS('8.2.0.0') and isCurrentOneFS('8.2.0.0') and $CLUSTER_INFO{'Build'} < 11; if (!hasPatch('^8\.2\.0.*(?:250838|254106|254793|254798|255568|257480|257594|259123|261104|262893|264782)')) { return } local $patch = '8.2.0_UDA-RUP_2020-02_264782'; local $re_versions = qr/^8\.2\.0\.0$/; local @requires = ('8.2.0_UGA-PATCH-INFRA_2019-08_251917'); local @replaceable_patches = hasPatch('^8\.2\.0_.*(?:250837|250838|254105|254106|254792|254793|254798|254807|254894|255567|255568|256081|256855|256885|257480|257593|257594|257853|259122|259123|259916|261103|261104|262892|262893|264781)$'); local @conflicting_patches = hasPatch('^8\.2\.0_.*(?:252624|254695|260301|264647|266617|266990)$'); local $impacts = "During installation of removal of $patch, the following services will be stopped and restarted automatically: apache2, auth, celog, dmilog, hdfs, ifsd, isi_audit_cee, isi_audit_d, isi_audit_syslog, isi_avscan_d, isi_boot_d, isi_cbind_d, isi_comp_d, isi_cpool_d, isi_cpool_io_d, isi_drive_d, isi_drive_repurpose_d, isi_drivemon, isi_esrs_d, isi_flexnet_d, isi_group_change_d, isi_hardening_d, isi_healthcheck, isi_hwmon, isi_ipmi_d, isi_job_d, isi_join_d, isi_km_d, isi_lbfo_d, isi_mcp, isi_migrate, isi_object_d, isi_papi_d, isi_ph_rpcd, isi_protohealth_d, isi_rpc_d, isi_rsapi_d, isi_sasphymon, isi_smartconnect, isi_smartpools_d, isi_smartquotas, isi_snapshot_d, isi_stats_d, isi_stats_hist_d, isi_tardis_d, isi_tardis_gcfg_d, isi_upgrade_agent_d, isi_upgrade_d, isi_vasa_d, isi_vc_d, isi_webui, lwsm, lwswift, ndmpd, nfs, pcnfsd, scheduler, smb, snmp, syslogd, telnetd, xinetd"; local $service_restart = 2; local $priority = 10; checkPatch(); } sub checkPatch821DA { if (not isDestinationOneFS('8.2.1.0') and isCurrentOneFS('8.2.1.0') and $CLUSTER_INFO{'Build'} < 4) { return } if (!hasPatch('^8\.2\.1.*(?:50|62|72|260|261|274|275|310|590|424|259120|261101|261351|262890|264779|266225|266915|268537|270424|272583)$')) { return } local $patch = '8.2.1_DA-RUP_2021-01_PSP-590'; local $re_versions = qr/^8\.2\.1\.0/; local @reboot = @LNNS; local $priority = 5; local @conflicting_patches = hasPatch('^8\.2\.1.*[-_](?:52|69|166|260302|266771|271509|273131|273703|273852)$'); local @replaceable_patches = hasPatch('^8\.2\.1.*[-_](?:50|62|72|260|261|274|275|310|397|424|257607|259119|259120|259121|261100|261101|261102|261351|262889|262890|262891|264778|264779|264780|264877|265633|266224|266225|266226|266915|268536|268537|268538|269272|270423|270424|271249|271478|272580|272583)$'); checkPatch(); } sub checkPatch821GA { if (not isDestinationOneFS('8.2.1.0') and isCurrentOneFS('8.2.1.0') and $CLUSTER_INFO{'Build'} < 4) { return } if (hasPatch('^8\.2\.1.*(?:50|62|72|260|261|274|275|310|590|424|259120|261101|261351|262890|264779|266225|266915|268537|270424|272583)$')) { return } local $patch = '8.2.1_GA-RUP_2021-01_PSP-397'; local $re_versions = qr/^8\.2\.1\.0/; local @reboot = @LNNS; local $priority = 5; local @conflicting_patches = hasPatch('^8\.2\.1.*[-_](?:50|52|62|69|72|166|260|261|274|275|310|424|259120|260302|261101|261351|262890|264779|266225|266771|266915|268537|270424|271509|272583|273131|273703|273852)$'); local @replaceable_patches = hasPatch('^8\.2\.1.*[-_](?:257607|259119|259121|261100|261102|262889|262891|264778|264780|264877|265633|266224|266226|268536|268538|269272|270423|271249|271478|272580)$'); checkPatch(); } sub checkPatch821HC { local $patch = 'HealthCheck_8.2.1_2021-09'; local $re_versions = qr/^8\.2\.1\.0$/; local @replaceable_patches = hasPatch('^HealthCheck_8.2.1_(?:2019|2020|2021-0[1235])'); checkPatch(); } sub checkPatch822DA { if (!hasPatch('8\.2\.2.*[-_](?:41|94|201|327|365|438|586|684|702|793|861|953|1032|1091|1150|1208|1243|1251|1308|1397|1493|1499|1687|258480|266222|267832|270135|271965)$')) { return } local $patch = '8.2.2_DA-RUP_2021-09_PSP-1687'; local $re_versions = qr/^8\.2\.2\.0/; local $priority = 10; local @reboot = @LNNS; local @replaceable_patches = hasPatch('^8\.2\.2.*[-_](?:40|41|93|94|99|101|200|201|326|327|365|386|437|438|507|514|585|586|683|684|702|716|793|794|839|860|861|903|952|953|1032|1088|1091|1092|1149|1150|1208|1243|1250|1251|1308|1396|1397|1490|1493|1498|1499|258480|258514|266221|266222|266223|267822|267830|267832|270134|270135|271964|271965)$'); local @conflicting_patches = hasPatch('^8\.2\.2.*[-_](?:382|735|821|922|1207|1491|270717|274246)$'); checkPatch(); } sub checkPatch822GA { if (hasPatch('8\.2\.2.*[-_](?:41|94|201|327|365|438|586|684|702|793|861|953|1032|1091|1150|1208|1243|1251|1308|1397|1493|1499|1687|258480|266222|267832|270135|271965)$')) { return } local $patch = '8.2.2_GA-RUP_2021-11_PSP-1571'; local $re_versions = qr/^8\.2\.2\.0/; local $priority = 10; local @reboot = @LNNS; local @replaceable_patches = hasPatch('^8\.2\.2.*[-_](?:40|93|99|101|200|326|386|437|507|514|585|683|716|794|839|860|903|952|1088|1092|1149|1250|1396|1490|1498|258514|266221|266223|267822|267830|270134|271964)$'); local @conflicting_patches = hasPatch('^8\.2\.2.*[-_](?:41|94|201|327|365|382|438|586|684|735|793|821|861|953|1032|1091|1150|1207|1208|1243|1251|1308|1397|1491|1493|1499|1611|1612|1642|1681|1686|1687|1717|258480|266222|267832|270135|271965)$'); checkPatch(); } sub checkPatch822HC { local $patch = 'HealthCheck_8.2.2_2021-09'; local $re_versions = qr/^8\.2\.2\.0$/; local @replaceable_patches = hasPatch('^HealthCheck_8.2.2_(?:2020|2021-0[1235])'); checkPatch(); } sub checkPatch900DA { if (!hasPatch('9\.0\.0.*[-_](?:68|171|278|407)$')) { return } local $patch = '9.0.0_DA-RUP_2020-11_PSP-407'; local $re_versions = qr/^9\.0\.0\.0/; local $priority = 10; local @reboot = @LNNS; local @replaceable_patches = hasPatch('^9\.0\.0.*[-_](?:38|67|68|170|171|277|278|406)$'); checkPatch(); } sub checkPatch900GA { if (hasPatch('9\.0\.0.*[-_](?:68|171|278|407)$')) { return } local $patch = '9.0.0_GA-RUP_2020-11_PSP-406'; local $re_versions = qr/^9\.0\.0\.0/; local $priority = 10; local @reboot = @LNNS; local @conflicting_patches = hasPatch('^9\.0\.0.*[-_](?:68|171|278|407)$'); local @replaceable_patches = hasPatch('^9\.0\.0.*[-_](?:38|67|170|277)$'); checkPatch(); } sub checkPatch900HC { local $patch = 'HealthCheck_9.0.0_2021-09'; local $re_versions = qr/^9\.0\.0\.0$/; local @replaceable_patches = hasPatch('^HealthCheck_9.0.0_(?:2020|2021-0[12356])'); checkPatch(); } sub checkPatch910HC { local $patch = 'HealthCheck_9.1.0_2021-09'; local $re_versions = qr/^9\.1\.0\./; local @replaceable_patches = hasPatch('^HealthCheck_9.1.0_(?:2020|2021-0[12356])'); checkPatch(); } sub checkPatch910Patch { if (hasPatch('9.1.0_UGA-PATCH-INFRA_2021-12_PSP-902')) { return } local $patch = '9.1.0_UGA-PATCH-INFRA_2021-12_PSP-902'; local $re_versions = qr/^(9\.1\.0)/; local $priority = 15; checkPatch(); } sub checkPatch920HC { local $patch = 'HealthCheck_9.2.0_2021-09'; local $re_versions = qr/^9\.2\.0/; local @replaceable_patches = hasPatch('^HealthCheck_9.2.0_(?:2021-0[356])'); checkPatch(); } sub checkPatch921HC { local $patch = 'HealthCheck_9.2.1_2021-09'; local $re_versions = qr/^9\.2\.1/; local @replaceable_patches = hasPatch('^HealthCheck_9.2.1_(?:2021-0[56])'); checkPatch(); } # checkPatchesEncodingCheck: Checks for EncodingCheck job patches installed sub checkPatchesEncodingCheck { if (wasCalled()) { return } # Setup my $SPF_ENCODING = 'Prior to performing an OneFS upgrade, patch %s, must be uninstalled'; # Analyze my ($encoding_patch) = hasPatch('patch-(185213|185212|185211|185210|185209|185207|185204'. '|185203|176073|176331|176068|176332|176069|176334|176335'. '|176336|176236|176070|176461|176460|176459|176458)'); # Flag if (isDestinationOneFS() and $encoding_patch) { critical(sprintf($SPF_ENCODING, $encoding_patch)); } } # End checkPatchesEncodingCheck # checkPatchesInstalling: Checks if any patches are installing sub checkPatchesInstalling { if (wasCalled()) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_gather_cluster_info -p]. q[| awk "/'installing'/,/\]/ {print}" ]; my $CMD_LOGS = q[ 2>&1 awk '//,/<\/installing>/ {print}' ]. q[ local/node_info_cache.tar/ifs/_ifsvar/db/node_info_cache/*/*.xml ]; my $CMD_LOGS2 = q[2>&1 grep -EH '/var/patch/tmp/[0-9a-f]+$' */du ]; my $SPF_PATCHES = q[The following patches were identified in isi_gather_cluster_info]. q[ -p as installing: %s]; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my %patches_installing = (); foreach my $line (@$gather) { my @patches = ($line =~ /'([^']+)'/g); foreach my $patch (@patches) { if ($patch eq 'installing') { next } if (hasPatch($patch)) { next } $patches_installing{$patch} = 1; } } # In case of a false positive in logs, check elsewhere if (!isLive() and %patches_installing) { $gather = gatherCluster($CMD_LOGS2); if (!%$gather) { %patches_installing = () } } # Flag if (%patches_installing) { fail(sprintf($SPF_PATCHES, join(', ', sort keys %patches_installing))); } } # End checkPatchesInstalling # checkPatchesNotInstalled: Check for patches not in an Installed status sub checkPatchesNotInstalled { if (wasCalled()) { return } # Setup my $SPF_NOT_INSTALLED = 'The following patches are reporting %s: %s'; # Analyze my %not_installed = (); foreach my $patch (hasPatch()) { if (!$CLUSTER_INFO{'Patches'}{$patch}{'Status'}) { next } my $status = $CLUSTER_INFO{'Patches'}{$patch}{'Status'}; if ($status ne 'Installed') { push(@{$not_installed{$status}}, $patch); } } # Flag if (%not_installed) { foreach my $status (keys %not_installed) { fail(sprintf($SPF_NOT_INSTALLED, $status, join(', ', @{$not_installed{$status}}))); } } } # End checkPatchesNotInstalled # checkPatchesPartial: Check for partially installed patches sub checkPatchesPartial { if (wasCalled()) { return } # Setup my $SPF_PARTIAL = 'The following patches are not installed on all nodes: %s'; # Analyze my $all_nodes = compressRange(@LNNS); my @partially_installed = (); foreach my $patch (hasPatch()) { if (!$CLUSTER_INFO{'Patches'}{$patch}{'Nodes'}) { next } my @nodes = @{$CLUSTER_INFO{'Patches'}{$patch}{'Nodes'}}; my $patched_nodes = compressRange(@nodes); if ($patched_nodes ne $all_nodes) { push(@partially_installed, "$patch ($patched_nodes)"); } } # Flag if (@partially_installed) { fail(sprintf($SPF_PARTIAL, join(', ', @partially_installed))); } } # End checkPatchesPartial $CHECKS{'checkPatches'}{'Description'} = 'Checks for highly recommended'. ' patches, on the current version if no target version, or for the target'. ' version.'; sub checkPatches { if (wasCalled()) { return } checkKB521778(); checkPartitions(); printCheck('Highly Recommended Patches'); gatherPatches(); gatherNetworkInterfaces(); my $SPF_ROOT_FIX = 'To avoid encountering errors during the installation of %s, the latest patch infrastructure patch, %s, must be installed prior.'; my $SPF_ROOT_BAD = 'The following nodes have been identified as having less than 1GB of space allocated to the root [/] partition: %s. These nodes may encounter errors during patch installs, see KB 000174169 (https://www.dell.com/support/kbdoc/000174169) for more information.'; my $MSG_PARALLEL = 'Parallel method for patch installs was requested, but is not available prior to OneFS 9.1 release, rolling method will be used'; # Display all installed patches my @patches = hasPatch(); if (@patches) { details("Installed Patches: @patches") } # Check for partially installed patches checkPatchesPartial(); # Check patches reporting a status other than Installed checkPatchesNotInstalled(); # Check if any temporary patches are installed checkPatchesTemporary(); # Check if any of the EncodingCheck job patches are installed checkPatchesEncodingCheck(); # Check if Patch 208294 is installed but missing /lib/libcrypto.so.8.patched checkKB520799(); # Check if any patches are listed as installing checkPatchesInstalling(); # Do not make patch recommendations when called via HealthCheck if ($RPS{'Patches'} eq 'None') { printResults(); return; } # Check for patches checkPatchSupportTools9x(); checkPatchSupportTools8x(); checkPatchRelease(); checkPatch921HC(); checkPatch920HC(); checkPatch910Patch(); checkPatch910HC(); checkPatch900DA(); checkPatch900GA(); checkPatch900HC(); checkPatch822GA(); checkPatch822DA(); checkPatch822HC(); checkPatch821GA(); checkPatch821DA(); checkPatch821HC(); checkPatch820Patch(); checkPatch820GA(); checkPatch820Kernel(); checkPatch820Userspace(); checkPatch820HC(); checkPatch813HC(); checkPatch813Userspace(); checkPatch813Kernel(); checkPatch812HC(); checkPatch8120GA(); checkPatch8120DA(); checkPatch8120Patch(); checkPatch810HC(); checkPatch8104Userspace(); checkPatch8104Kernel(); checkPatch8007HC(); checkPatch8007Userspace(); checkPatch8007Kernel(); checkPatch231386(); checkPatch229160(); checkPatch229158(); checkPatch229156(); checkPatch228458(); checkPatch226984(); checkPatch226323(); checkPatch224116(); checkPatch223338(); checkPatch223311(); checkPatch222487(); checkPatch221949(); checkPatch221947(); checkPatch221603(); checkPatch221602(); checkPatch220176(); checkPatch220098(); checkPatch219279(); checkPatch213287(); checkPatch213286(); checkPatch206322(); checkPatch189133(); checkPatch189132(); checkPatch188660(); checkPatch188239(); checkPatch181490(); checkPatch179385(); checkPatch178793(); checkPatch175451(); checkPatch164666(); checkPatch161166(); checkPatch147686(); checkPatch146974(); # If there is no OneFS upgrade being performed if (!isOneFSUpgrade() and $RPS{'Post Upgrade Patches'}) { # Check if applying patches and nodes have less than 1GB allocated to / if ($RPS{'Root less than 1GB'}) { my ($patch) = grep(/_[DG]A-RUP_/, keys %{$RPS{'Post Upgrade Patches'}}); if ($patch) { if (isCurrentOneFS('8.1.2.0')) { my $infra_patch = '8.1.2.0_UGA-PATCH-INFRA_2020-08_PSP-268'; if (!hasPatch($infra_patch)) { critical(sprintf($SPF_ROOT_FIX, $patch, $infra_patch)); } } elsif (isCurrentOneFS(qr/^9\.1\.0/)) { my $infra_patch = '9.1.0_UGA-PATCH-INFRA_2021-12_PSP-902'; if (!hasPatch($infra_patch)) { critical(sprintf($SPF_ROOT_FIX, $patch, $infra_patch)); } } elsif (isCurrentOneFS(qr/^(?:8\.2|9\.0)\./)) { fail(sprintf($SPF_ROOT_BAD, compressRange(keys %{$RPS{'Root less than 1GB'}}))); } } } # The patches should be Pre Upgrade Patches, not Post Upgrade Patches. if (!$RPS{'Pre Upgrade Patches'}) { $RPS{'Pre Upgrade Patches'} = $RPS{'Post Upgrade Patches'}; } else { foreach my $patch (keys %{$RPS{'Post Upgrade Patches'}}) { next if $RPS{'Pre Upgrade Patches'}{$patch}; $RPS{'Pre Upgrade Patches'}{$patch} = $RPS{'Post Upgrade Patches'}{$patch}; } } delete $RPS{'Post Upgrade Patches'}; # Flag that parallel was requested but not available if (isCurrentOneFS('<9.1') and exists $RPS{'Patch'}{'Method'} and $RPS{'Patch'}{'Method'} eq 'parallel') { warning($MSG_PARALLEL); $RPS{'Patch'}{'Method'} = 'rolling'; } } printResults(); } # End checkPatches # checkPatchesTemporary: Checks if any 'temporary' patches are installed sub checkPatchesTemporary { if (wasCalled()) { return } # Setup my $SPF_TMP_PATCHES = 'The following packages should be uninstalled: %s'; # Analyze my @temporary_patches = hasPatch('IsiFru_Package|IsiHw_Package'. '|Drive_Support|Drive_Firmware|IsiFw'); # Flag if (@temporary_patches) { critical(sprintf($SPF_TMP_PATCHES, join(', ', @temporary_patches))); } } # End checkPatchesTemporary sub checkPatchRelease { if (wasCalled()) { return } if (!isDestinationOneFS('>=9.1')) { return } local $target = isDestinationOneFS(); if (!$Patch_Releases{$target}) { return } local $patch = $Patch_Releases{$target}{'GA-RUP'}; # Extract from destination the release specs my ($major, $minor, $service) = split(/\./, $target); local $re_versions = qr/^$major\.$minor\.$service\./; # If only a patch upgrade and there are DA-RUP/CUSTOM/RESTRICTED patches if (isCurrentOneFS($re_versions) and hasPatch('DA-RUP|CUSTOM|RESTRICTED')) { # Exceptions for DA RUPs that are deprecated by the GA RUP if ((hasPatch('_PSP-945$') and isDestinationOneFS('>=9.1.0.5')) or (hasPatch('_PSP-1022$') and isDestinationOneFS('>=9.1.0.6')) or (hasPatch('_PSP-1080$') and isDestinationOneFS('>=9.1.0.7')) or (hasPatch('_PSP-1110$') and isDestinationOneFS('>=9.1.0.9')) or (hasPatch('_PSP-1289$') and isDestinationOneFS('>=9.1.0.10')) or (hasPatch('_PSP-1523$') and isDestinationOneFS('>=9.1.0.11')) or (hasPatch('_PSP-1608$') and isDestinationOneFS('>=9.1.0.12')) or (hasPatch('_PSP-1312$') and isDestinationOneFS('>=9.2.0.1'))) { } elsif (defined $Patch_Releases{$target}{'DA-RUP'}) { $patch = $Patch_Releases{$target}{'DA-RUP'}; } } local $priority = 10; local @reboot = @LNNS; checkPatch(); } # Uses local variables to validate patches sub checkPatch { if (!$patch or !$re_versions) { return } if (isDestinationOneFS()) { if (!isDestinationOneFS($re_versions)) { return } # For patch releases when the destination patch is already installed if (isCurrentOneFS('>=9.1') and isCurrentOneFS($re_versions) and hasPatch($patch)) { return; } } else { if (!isCurrentOneFS($re_versions) or hasPatch($patch)) { return } } $RPS{'Post Upgrade Patches'}{$patch} = (); $RPS{'Post Upgrade Patches'}{$patch}{'Impacts'} = $impacts if $impacts; $RPS{'Post Upgrade Patches'}{$patch}{'Important'} = $important if $important; $RPS{'Post Upgrade Patches'}{$patch}{'Service Restart'} = $service_restart if $service_restart; $RPS{'Post Upgrade Patches'}{$patch}{'Priority'} = $priority if $priority; $RPS{'Post Upgrade Patches'}{$patch}{'Target'} = $target if $target; @{$RPS{'Post Upgrade Patches'}{$patch}{'Requires'}} = @requires if @requires; @{$RPS{'Post Upgrade Patches'}{$patch}{'Reboot'}} = @reboot if @reboot; if (isDestinationOneFS()) { warning("$patch is recommended for the destination OneFS $RPS{'Target'}."); } else { warning("$patch is recommended for the current OneFS $CLUSTER_INFO{'OneFS'}."); if (@conflicting_patches) { warning("$patch is recommended but is known to conflict with patches: ".join(", ", @conflicting_patches)); warning("If the customer would like to have $patch performed they should review the conflicting patches and determine whether they want to keep them or perform $patch"); } if (@deprecated_patches) { warning("Prior to installing $patch, the following deprecated patches must be uninstalled: ".join(", ", @deprecated_patches)); } if (@replaceable_patches) { warning("During the installation of $patch, the following deprecated patches will be replaced: ".join(", ", @replaceable_patches)); } foreach my $other_patch (@deprecated_patches, @conflicting_patches) { if ($patches_with_svc_scripts and $other_patch =~ $patches_with_svc_scripts) { $RPS{'Post Upgrade Patches'}{"uninstall $other_patch"}{'Service Restart'} = 1 if $service_restart; } else { $RPS{'Post Upgrade Patches'}{"uninstall $other_patch"}{'Service Restart'} = $service_restart if $service_restart; } $RPS{'Post Upgrade Patches'}{"uninstall $other_patch"}{'Impacts'} = "Please review the README for $other_patch for details on impacts during removal"; @{$RPS{'Post Upgrade Patches'}{"uninstall $other_patch"}{'Reboot'}} = @reboot if @reboot; push(@{$RPS{'Post Upgrade Patches'}{$patch}{'Requires'}}, "uninstall $other_patch"); } } } sub checkPatchSupportTools8x { local $patch = '8.X_GA-TOOLS_2021-05_PSP-1034'; local $re_versions = qr/^(?:8\.1\.2|8\.2\.2)/; local $priority = 1; local @requires = ('8.1.2.0_UGA-PATCH-INFRA_2019-09_255624'); local @replaceable_patches = hasPatch('PSP-(?:1034|587)$'); checkPatch(); # If support tools package PSP-587 is installed, it must be uninstalled # prior to an OneFS upgrade my ($tools_patch) = hasPatch('PSP-587$'); if ($tools_patch and isCurrentOneFS('<9') and isOneFSUpgrade()) { critical("Prior to performing an OneFS upgrade, the support tools package, $tools_patch, must be uninstalled"); $RPS{'Pre Upgrade Patches'}{"uninstall $tools_patch"} = (); } } sub checkPatchSupportTools9x { local $patch = '9.X_GA-TOOLS_2021-10_PSP-1695'; local $re_versions = qr/^(?:9\.)/; local $priority = 1; local @replaceable_patches = hasPatch('PSP-(?:1306|1034|587)$'); checkPatch(); # If support tools package PSP-587 is currently installed, it must be # uninstalled prior to an OneFS upgrade, however, that would cause service # interruptions. To avoid those interruptions, PSCALE-107422, the latest # support tools package should first be installed my ($tools_patch) = hasPatch('PSP-587'); if ($tools_patch and isCurrentOneFS('>=9') and isOneFSUpgrade()) { critical("Prior to performing an OneFS upgrade, the support tools package, $tools_patch, must be replaced with the latest support tools package, $patch"); $RPS{'Pre Upgrade Patches'}{$patch} = (); } } ### # gatherPatchesCluster8: Gathers a list of installed patches. sub gatherPatchesCluster8 { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return } # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi upgrade patches list -v ]. q[ | grep -E 'Patch Name:|Status:|^-+$' ]); } else { $gather = gatherLocal(q[2>&1 grep -E 'Patch Name:|Status:|^-+$' local/patch_list]); } # Assign CLUSTER_INFO based on gather my $patch = ""; foreach my $line ($gather) { $patch = $1 if $line =~ /Patch Name:\s+(\S+)$/; $patch = "" if $line =~ /^-+$/; next unless $patch; if ($line =~ /Status:\s+(\S.*)$/) { $CLUSTER_INFO{'Patches'}{$patch}{'Status'} = $1; debug("CLUSTER_INFO=>Patches=>${patch}=>Status=$1"); } } } # End gatherPatchesCluster8 ### # gatherPatchesLocal7: Gathers a list of installed patches. sub gatherPatchesLocal7 { if (wasCalled()) { return } if (isCurrentOneFS('>=8')) { debug("gatherPatchesLocal7: Skipping. OneFS version 8.0 or later"); return; } # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array "isi pkg info --forced_local | grep '^[[:alnum:]]'"]); } else { $gather = gatherCluster(qq[2>&1 grep '^[[:alnum:]]' */packages_local]); } # Assign CLUSTER_INFO based on gather my %patches = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /^\s*(\S+):/; push(@{$patches{$1}}, $lnn); } } foreach my $patch (keys %patches) { @{$CLUSTER_INFO{'Patches'}{$patch}{'Nodes'}} = @{$patches{$patch}}; debug("CLUSTER_INFO=>Patches=>${patch}=".join(",", @{$CLUSTER_INFO{'Patches'}{$patch}{'Nodes'}})); } } # End gatherPatchesLocal7 ### # gatherPatchesLocal8: Gathers a list of installed patches. sub gatherPatchesLocal8 { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { debug("gatherPatchesLocal8: Skipped, version prior to OneFS 8"); return; } # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array isi upgrade patches list -v --local | grep -E 'Patch Name:']); } else { $gather = gatherCluster(qq[2>&1 grep -E 'Patch Name:' */patch_list_local]); } # Assign CLUSTER_INFO based on gather my %patches = (); foreach my $lnn (@LNNS) { foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /^\s*Patch Name: (\S+)/; push(@{$patches{$1}}, $lnn); } } foreach my $patch (keys %patches) { @{$CLUSTER_INFO{'Patches'}{$patch}{'Nodes'}} = @{$patches{$patch}}; debug("CLUSTER_INFO=>Patches=>${patch}=".join(",", @{$CLUSTER_INFO{'Patches'}{$patch}{'Nodes'}})); } } # End gatherPatchesLocal8 ### # gatherPatches: Gathers a list of installed patches. sub gatherPatches { return if wasCalled(); gatherPatchesLocal8(); gatherPatchesCluster8(); gatherPatchesLocal7(); } # End gatherPatches ### # hasPatch: Checks if the cluster has installed a patch that matches the regex sub hasPatch { my ($regex, $lnn) = @_; # Gather patches gatherPatches(); # Return an empty list if there are no patches if (!$CLUSTER_INFO{'Patches'}) { return () } # Generate the list of patches my @patches = sort keys %{$CLUSTER_INFO{'Patches'}}; # Return all patches if no regex if (!$regex) { return @patches } # Generate the list of patches that match the regex my @matched_patches = grep /$regex/, @patches; # Return patches that match the provided regex if no lnn if (!$lnn) { return @matched_patches } # Generate the list of patches for the requested lnn that match the regex my @lnn_matched_patches = (); foreach my $pkg (@matched_patches) { if (!$CLUSTER_INFO{'Patches'}{$pkg}{'Nodes'}) { next } my @nodes = @{$CLUSTER_INFO{'Patches'}{$pkg}{'Nodes'}}; my ($match) = grep {$_ eq $lnn} @nodes; if ($match) { push(@lnn_matched_patches, $pkg) } } return @lnn_matched_patches; } # End hasPatch $CHECKS{'checkAspera'}{'Description'} = 'Checks if any aspera services are'. ' enabled. If performing OneFS upgrades it must be reinstalled'. ' post-upgrade.'; sub checkAspera { if (wasCalled()) { return } if (isCurrentOneFS('>8.1.2')) { return unsupported('Supported in OneFS 8.1.2 and earlier'); } if (!isDestinationOneFS()) { return unsupported('Supported when using the parameter target_version'); } # Setup printCheck('Aspera'); my $MSG_REINSTALL = 'If running IBM Aspera for Isilon, it must be'. ' reinstalled after OneFS upgrades to restore the Aspera service.'. ' Please refer to IBM Aspera documentation for more details.'; my $MSG_SUPPORT = 'If running IBM Aspera for Isilon, it is only supported'. ' up to OneFS 8.1.2. Please refer to IBM Aspera documentation for'. ' more details.'; # Flag if (isServiceEnabled('asperacentral') or isServiceEnabled('asperanoded')) { if (isDestinationOneFS('>8.1.2')) { addCustomerInformation(warning($MSG_SUPPORT)); } else { addCustomerInformation(warning($MSG_REINSTALL)); } } # Output printResults(); } ### # checkAuthStatus: Checks authentication status on each node. # Warns if any authentication providers are not online or active. $CHECKS{'checkAuthStatus'}{'Description'} = "Checks authentication status on each node. Warns if any authentication providers are not online or active. Checks for RFC2307 and Auto-locate of GID/UIDs and points to KB335338"; sub checkAuthStatus { if (wasCalled()) { return } printCheck("Authentication Status"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { if (isCurrentOneFS('>=7')) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi auth status --verbose --format list"); } else { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array isi auth status --verbose"); } } else { $gather = gatherCluster("2>&1 grep -H . */isi_auth_ads_status */isi_auth"); } # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } my $id = ""; foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /^\s*Domain:.*?(\S+)/ or $line =~ /^\s*ID:.*?(\S+)/) { $id = $1; } next unless $id; if ($line =~ /^\s*Status:.*?(\S+)/) { my $status = $1; verbose("Provider:$id Status:$status"); warning("On node $lnn, authentication provider $id is $status") if ($status !~ /active|online|disabled/); $id = ""; } } } checkRFC2307(); info("Refer to KB 335106 (https://www.dell.com/support/kbdoc/335106) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkAuthStatus $CHECKS{'checkCapacity'}{'Description'} = "Verifies cluster capacity based on documented numbers the Upgrade Planning and Progress Guide. Warns if approaching."; sub checkCapacity { if (wasCalled()) { return } printCheck("Cluster Capacity"); gatherIsiStatPools(); gatherStat(); verboseHeader("Cluster Capacity"); verbose("Cluster VHS Size: $CLUSTER_INFO{'VHS Size'}") if $CLUSTER_INFO{'VHS Size'}; verbose("Cluster HDD Storage: $CLUSTER_INFO{'HDD Storage'}") if $CLUSTER_INFO{'HDD Storage'}; verbose("Cluster HDD Used: $CLUSTER_INFO{'HDD Used'}") if $CLUSTER_INFO{'HDD Used'}; verbose("Cluster SSD Storage: $CLUSTER_INFO{'SSD Storage'}") if $CLUSTER_INFO{'SSD Storage'}; verbose("Cluster SSD Used: $CLUSTER_INFO{'SSD Used'}") if $CLUSTER_INFO{'SSD Used'}; my @pools_over_90_percent = (); my @pools_over_88_percent = (); my @all_pools = (); debug("checkCapacity: Checking pool usages"); if ($CLUSTER_INFO{'Pools'}) { foreach my $name (keys %{$CLUSTER_INFO{'Pools'}}) { my %pool = %{$CLUSTER_INFO{'Pools'}{$name}}; verbose("Pool $name Health: $pool{'Health'}") if $pool{'Health'}; verbose("Pool $name Protection: $pool{'Protection'}") if $pool{'Protection'}; verbose("Pool $name VHS Size: $pool{'VHS'}") if $pool{'VHS'}; verbose("Pool $name Percent Used: $pool{'Percent Used'}%") if $pool{'Percent Used'}; verbose("Pool $name HDD Storage: $pool{'HDD Storage'}") if $pool{'HDD Storage'}; verbose("Pool $name SSD Storage: $pool{'SSD Storage'}") if $pool{'SSD Storage'}; next unless $pool{'Percent Used'}; push(@all_pools, $name); if ($pool{'Percent Used'} >= 90) { push(@pools_over_90_percent, $name); debug("checkCapacity: !!! Pool $name is over 90% used"); } elsif ($pool{'Percent Used'} >= 88) { push(@pools_over_88_percent, $name); } } } if (@pools_over_90_percent and (scalar(@pools_over_90_percent) eq scalar(@all_pools))) { fail("All pools above 90% capacity"); } if (isCurrentOneFS('>=8')) { foreach my $pool (@pools_over_90_percent) { fail("Pool $pool at $CLUSTER_INFO{'Pools'}{$pool}{'Percent Used'}% capacity"); } } else { foreach my $pool (@pools_over_90_percent) { warning("Pool $pool at $CLUSTER_INFO{'Pools'}{$pool}{'Percent Used'}% capacity"); } } foreach my $pool (@pools_over_88_percent) { warning("Pool $pool at $CLUSTER_INFO{'Pools'}{$pool}{'Percent Used'}% capacity"); } # Fail if total cluster usage is over 90%, warn if approaching if ($CLUSTER_INFO{'HDD Percent Used'} and $CLUSTER_INFO{'HDD Percent Used'} >= 90) { fail("Cluster is $CLUSTER_INFO{'HDD Percent Used'}% used");#, recommended minimum of available space on the cluster is 10%."); } elsif ($CLUSTER_INFO{'HDD Percent Used'} and $CLUSTER_INFO{'HDD Percent Used'} >= 88) { warning("Cluster is $CLUSTER_INFO{'HDD Percent Used'}% used");#, this is approaching the recommended minimum of available space on the cluster of 10%."); } # Check usage on each node foreach my $lnn (@LNNS) { if ($NODE_INFO{$lnn}{'SSD Storage'} and $NODE_INFO{$lnn}{'SSD Storage'} =~ /(\d+)%/) { if ($1 >= 92) { fail("Node $lnn SSD Capacity is at ${1}%"); } elsif ($1 >= 88) { warning("Node $lnn SSD Capacity is at ${1}%"); } } next unless $NODE_INFO{$lnn}{'Percent Used'}; verbose("Node $lnn is at $NODE_INFO{$lnn}{'Percent Used'}% capacity"); if ($NODE_INFO{$lnn}{'Percent Used'} >= 92) { fail("Node $lnn is at $NODE_INFO{$lnn}{'Percent Used'}% capacity");#, recommended minimum amount of available space for each node is 8%."); } elsif ($NODE_INFO{$lnn}{'Percent Used'} >= 88) { warning("Node $lnn is at $NODE_INFO{$lnn}{'Percent Used'}% capacity");#, this is approaching the recommended minimum amount of available space for each node of 8%."); } } addCustomerInformation("Per the 'Check for free node space' section of the Upgrade Planning and Process Guide (https://dl.dell.com/content/docu69989): The total amount of used space on the entire cluster must not exceed 90 percent full, and the total amount of used space on each node must not exceed 92 percent full. Do not continue with the OneFS upgrade if the minimum available-space requirement for the cluster is not met. For more information, see the Best Practices Guide for Maintaining Enough Free Space on Isilon Clusters and Pools (https://dl.dell.com/content/docu48119).") if isResults(&WARN); info("Refer to KB 335107 (https://www.dell.com/support/kbdoc/335107) for details.") if isResults(&WARN); printResults(); } # End checkCapacity $CHECKS{'checkContact'}{'Description'} = "Checks contact information configured in CELog (only useful when called with --extra)"; sub checkContact { if (wasCalled()) { return } printCheck("Contact Information"); gatherLicense(); verboseHeader("Contact Information"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig | grep contact_info]); } elsif (-e "local/isi_gconfig") { $gather = gatherLocal(qq[2>&1 grep contact_info local/isi_gconfig]); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { $gather = gatherLocal(qq[2>&1 grep contact_info local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]); } # Analyze information for issues. my $secondary_name = ""; foreach my $line (@$gather) { if ($line =~ /celog.contact_info.company.*?= \"?(.*?)\"?$/) { verbose("Company Name: $1"); } elsif ($line =~ /celog.contact_info.location.*?= \"?(.*?)\"?$/) { verbose("Location: $1"); } elsif ($line =~ /celog.contact_info.primary_name.*?= \"?(.*?)\"?$/) { verbose("Primary Name: $1"); } elsif ($line =~ /celog.contact_info.primary_phone1.*?= \"?(.*?)\"?$/) { verbose("Primary Phone: $1"); } elsif ($line =~ /celog.contact_info.primary_email.*?= \"?(.*?)\"?$/) { verbose("Primary E-mail: $1"); } elsif ($line =~ /celog.contact_info.secondary_name.*?= \"?(.*?)\"?$/) { verbose("Secondary Name: $1"); $secondary_name = $1; } elsif ($line =~ /celog.contact_info.secondary_phone1.*?= \"?(.*?)\"?$/) { verbose("Secondary Phone: $1"); } elsif ($line =~ /celog.contact_info.secondary_email.*?= \"?(.*?)\"?$/) { verbose("Secondary E-mail: $1"); } } if ($CLUSTER_INFO{'OneFS License ID'} and isCurrentOneFS(qr/^8\.1\.1\.0$|^8\.1\.0\.[012]$/) and $secondary_name !~ /$CLUSTER_INFO{'OneFS License ID'}/) { warning("Isilon SWID not found in the Secondary Name field, see KB517765 for instructions on how to add it"); } printResults(); } # End checkContact $CHECKS{'checkEmailSettings'}{'Description'} = "Checks Email settings configured in CELog (only useful when called with --extra)"; sub checkEmailSettings { return if wasCalled(); printCheck("Email Settings"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig | grep "celog_settings" ]); } elsif (-e "local/isi_gconfig") { $gather = gatherLocal(qq[2>&1 grep "celog_settings" local/isi_gconfig]); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { $gather = gatherLocal(qq[2>&1 grep "celog_settings" local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]); } verboseHeader("Email Settings"); # Analyze information for issues. foreach my $line (@$gather) { if ($line =~ /celog\.celog_settings\.mail_relay.*?= \"?(.*?)\"?$/) { verbose("SMTP Relay Address: $1"); } elsif ($line =~ /celog\.celog_settings\.smtp_port.*?= \"?(.*?)\"?$/) { verbose("SMTP Relay Port: $1"); } elsif ($line =~ /celog\.celog_settings\.mail_sender.*?= \"?(.*?)\"?$/) { verbose("Send E-mail as: $1"); } elsif ($line =~ /celog\.celog_settings\.mail_subject.*?= \"?(.*?)\"?$/) { verbose("Subject Line: $1"); } elsif ($line =~ /celog\.celog_settings\.use_smtp_auth.*?= \"?(.*?)\"?$/) { verbose("Use SMTP Authentication: $1"); } } printResults(); } # End checkEmailSettings ### # checkEncoding: Checks exports and cluster configuration for # if utf-8/default encoding. $CHECKS{'checkEncoding'}{'Description'} = "Checks exports and cluster configuration for if utf-8/default encoding."; sub checkEncoding { if (wasCalled()) { return } printCheck("Cluster Encoding"); verboseHeader("Cluster Encoding"); gatherArrayXML(); gatherNFSExport(); verbose("Cluster Encoding: $CLUSTER_INFO{'Encoding'}") if $CLUSTER_INFO{'Encoding'}; my @ids = (); foreach my $ID (sort {$a <=> $b} keys %{$CLUSTER_INFO{'NFS Export'}}) { next unless ($CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'}); verbose("NFS Export ID $ID Encoding: $CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'}"); push(@ids, $ID) if ($CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'} !~ /DEFAULT|UTF-8/i); } if (isCurrentOneFS('<7.2') and isDestinationOneFS('>=7.2') and isDestinationOneFS('<8.1.1')) { if ($CLUSTER_INFO{'Encoding'} and $CLUSTER_INFO{'Encoding'} !~ /utf-8/) { addCustomerInformation(fail("ETA483840: The cluster is using $CLUSTER_INFO{'Encoding'} encoding and should not be upgraded to 7.2 or later.")); } if (@ids) { addCustomerInformation(fail("ETA483840: The following NFS Export IDs: ".join(",",@ids)." use non-DEFAULT/UTF-8 encoding and should not be upgraded to 7.2 or later.")); } } info("Refer to KB 335104 (https://www.dell.com/support/kbdoc/335104) for details.") if isResults(&WARN); printResults($CLUSTER_INFO{'Encoding'}); } # End checkEncoding sub checkEventChannel { if (wasCalled()) { return } gatherEventChannel(); gatherRemoteSupport(); if (!$CLUSTER_INFO{'Event Channels'}) { return } my $MSG = 'RemoteSupport (isi remotesupport) is enabled, but the RemoteSupport event channel is disabled and should be enabled for DialHome functionality. To enable the RemoteSupport event channel run the following command: isi event channel modify RemoteSupport --enabled=true'; # Flag an issue where the RemoteSupport event channel is not enabled and # the cluster is configured for remotesupport my $channels = $CLUSTER_INFO{'Event Channels'}; my $disabled = 0; foreach my $id (sort keys %$channels) { if ($channels->{$id}{'Name'} eq 'RemoteSupport' and $channels->{$id}{'Enabled'} ne 'Yes') { $disabled = 1; } } if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and $disabled) { critical($MSG); } # Verbose output for event channels verboseHeader('Event Channels'); my $spf = '%-5s %-10s %-10s %s'; verbose(sprintf($spf, 'ID', 'Type', 'Enabled', 'Name')); verbose(sprintf($spf, '-'x5, '-'x10, '-'x10, '-'x20)); foreach my $id (sort keys %$channels) { verbose(sprintf($spf, $channels->{$id}{'ID'}, $channels->{$id}{'Type'}, $channels->{$id}{'Enabled'}, $channels->{$id}{'Name'})); } } $CHECKS{'checkEvents'}{'Description'} = "Checks events on all the nodes, failure if any critical events exist."; sub checkEvents { return if wasCalled(); printCheck("Critical Events"); gatherEvents(); # Analyze information for issues. foreach my $id (sort keys %{$CLUSTER_INFO{'Events'}}) { next unless $CLUSTER_INFO{'Events'}{$id}{'Severity'} and $CLUSTER_INFO{'Events'}{$id}{'Ended'} and $CLUSTER_INFO{'Events'}{$id}{'Lnn'} and $CLUSTER_INFO{'Events'}{$id}{'Message'}; if ($CLUSTER_INFO{'Events'}{$id}{'Severity'} =~ /^c/i and $CLUSTER_INFO{'Events'}{$id}{'Ended'} eq "--") { fail("Critical event $id for node ".join(",", @{$CLUSTER_INFO{'Events'}{$id}{'Lnn'}}).": ".join(",", @{$CLUSTER_INFO{'Events'}{$id}{'Message'}})); } } info("Refer to KB 335102 (https://www.dell.com/support/kbdoc/335102) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkEvents $CHECKS{'checkFileSharing'}{'Description'} = 'Checks if Atime is enabled'; sub checkFileSharing { if (wasCalled()) { return } # Setup printCheck('File Sharing'); my $CMD_LIVE = "2>&1 ${SUDO} sysctl efs.bam.atime_enabled". " efs.bam.atime_grace_period"; my $CMD_LOGS = '2>&1 grep -H atime'. ' local/ifsvar_etc.tar/etc/mcp/override/sysctl.conf'; my $MSG_HOUR = 'Access time tracking is enabled and the grace period is'. ' less than one hour'; my $MSG_DAY = 'Acess time tracking is enabled and the grace period is less'. ' than one day'; my $MSG_KB = 'For details see KB 303681'. ' [https://www.dell.com/support/kbdoc/en-us/303681 - Isilon:'. ' OneFS-How to enable access time tracking (atime)]'; my $MS_DAY = 86400000; my $MS_HOUR = 3600000; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze my $atime_enabled = 0; my $atime_grace_period = $MS_DAY; foreach (@$gather) { if (/atime_enabled.*?(\d+)$/) { $atime_enabled = $1; } if (/atime_grace_period.*?(\d+)$/) { $atime_grace_period = $1; } } # Flag if ($atime_enabled) { debug('checkFileSharing: access time tracking (atime) is enabled'); if ($atime_grace_period < $MS_HOUR) { critical($MSG_HOUR) } elsif ($atime_grace_period < $MS_DAY) { warning($MSG_DAY) } } if (isResults(&WARN)) { info($MSG_KB) } # Output printResults(); } $CHECKS{'checkFirmwarePackages'}{'Description'} = 'In OneFS 9.1 and later, confirms firmware packages are available'; sub checkFirmwarePackages { if (wasCalled()) { return } if (!$RPS{'Live'}) { return unsupported('Must be run on an Isilon cluster'); } if (isVirtual()) { return unsupported('Unsupported on virtual platforms'); } if (isCurrentOneFS('<9.1')) { return unsupported('Supported in OneFS 9.1 and later'); } # Setup printCheck('Firmware Packages'); my $SPF_MISSING = 'The cluster does not have the available firmware packages. To ensure'. ' the cluster is running qualified firmware, firmware packages must be'. ' present. Packages missing: %s'; my $MSG_DSP = 'To install drive firmware, download the latest Drive Support Package,'. ' and run the "isi_dsp_install " command.'; my $MSG_NFP = 'To install node firmware, download the latest Node Firmware Package,'. ' and run the "isi upgrade firmware assess --fw-pkg="'. ' command.'; my $CMD_LIVE = '2>&1 ls -1 /ifs/.ifsvar/packages/'; my $RE_DSP = qr/Drive_Support_v\d+(?:\.\d+)*/; my $RE_NFP = qr/IsiFw_Package_v\d+(?:\.\d+)*(?:qa\d+)?/; # Gather my $gather = gatherLocal($CMD_LIVE); # Analyze my $has_dsp = 0; my $has_nfp = 0; foreach my $line (@$gather) { if ($line =~ $RE_DSP) { $has_dsp = 1 } if ($line =~ $RE_NFP) { $has_nfp = 1 } } # Flag if (!$has_dsp and !$has_nfp) { warning(sprintf($SPF_MISSING, 'Drive Support Package, Node Firmware Package')); warning($MSG_DSP); warning($MSG_NFP); } elsif (!$has_dsp) { warning(sprintf($SPF_MISSING, 'Drive Support Package')); warning($MSG_DSP); } elsif (!$has_nfp) { warning(sprintf($SPF_MISSING, 'Node Firmware Package')); warning($MSG_NFP); } # Output printResults(); } # End checkFirmwarePackages $CHECKS{'checkGroups'}{'Description'} = "Checks nodes for all enabled protocols. Fails if group info is reporting that an enabled protocol is not functioning on any node."; sub checkGroups { if (wasCalled()) { return } return unless $RPS{'Live'}; return unless isCurrentOneFS('>=8'); printCheck("All Enabled Protocols"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster (qq[2>&1 ${SUDO} isi_for_array ]. qq[sysctl efs.gmp.service.all_enabled_protocols.pid]); } # Analyze information for issues. my %not_all_enabled_protocols = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } my @results = grep(/all_enabled_protocols.pid:\s+-1$/, @{$gather->{$lnn}}); $not_all_enabled_protocols{$lnn} = 1 if @results; } # Flag issues detected fail("Not all enabled protocols are functioning on the following nodes: ". compressRange(keys %not_all_enabled_protocols)) if %not_all_enabled_protocols; # Output Check result printResults(); } # End checkGroups sub checkHDFSAuthenticationMode { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2.2')) { return } my $CMD_LOGS = q[2>/dev/null egrep '^ *(HDFS settings for zone|Authentication Mode): *' local/isi_hdfs_zone_settings | sed -r -e 's/ *(HDFS settings for zone|Authentication Mode): *//' | paste -d, - -]; my $CMD_LIVE = qq[for z in \$(2>/dev/null ${SUDO} isi zone zone list --format list | egrep '^ *Name: *' | sed -r -e 's/ *Name: *//'); do a=\$(2>/dev/null ${SUDO} isi hdfs settings view --zone "\$z" | egrep '^ *Authentication Mode: *' | sed -r -e 's/ *Authentication Mode: *//'); echo "\$z,\$a"; done 2>/dev/null]; my $SPF_ZONES = 'HDFS license is present on cluster and the HDFS setting '. '"authentication_mode" is set to "all" for at least one access zone. '. 'If upgrading to version of OneFS 8.2.2 or higher, '. '"authentication_mode" setting value will automatically be migrated to '. '"simple_only". This automatic setting migration can lead to future '. 'client connectivity issues if access zones are configured to use '. 'kerberos, active directory authentication providers, or HDFS proxy '. 'users with HDFS enabled. The authentication_mode is set to all for '. 'the following access zones: %s'; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze information for issues my @zones = (); foreach (@$gather) { my ($zone, $mode) = split /,/; if (!$zone or !$mode) { next } if ($mode eq 'all') { push(@zones, $zone) } } # Return if no issue if (!@zones) { return } # Flag issues my $msg = sprintf($SPF_ZONES, join(', ', @zones)); warning($msg); if (isDestinationOneFS('>=8.2.2')) { addCustomerInformation($msg); } } # End checkHDFSAuthenticationMode $CHECKS{'checkHDFS'}{'Description'} = 'Display HDFS details, only useful when run with --extra'; sub checkHDFS { return if wasCalled(); gatherLicense(); if (!hasLicense('HDFS')) { return unsupported('No HDFS license found'); } printCheck('HDFS'); checkHDFSAuthenticationMode(); verboseHeader('HDFS'); my ($license_href) = getLicense('HDFS'); verbose("HDFS License Status: $license->{'Status'}") if $license->{'Status'}; verbose("HDFS License Expiration: $license->{'Expiration'}") if $license->{'Expiration'}; my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi_gconfig -t hdfs"); } elsif ( -e 'local/isi_gconfig_hdfs') { $gather = gatherLocal('2>&1 cat local/isi_gconfig_hdfs'); } elsif ( -e 'local/ifsvar_modules.tar/modules/hdfs_d/config/policy.gc') { $gather = gatherLocal('2>&1 cat local/ifsvar_modules.tar/modules/hdfs_d/config/policy.gc'); } my %lg_bln_sz = ( 12 => '4K', 13 => '8K', 14 => '16K', 15 => '32K', 16 => '64K', 17 => '128K', 18 => '256K', 19 => '512K', 20 => '1M', 21 => '2M', 22 => '4M', 23=> '8M', 24=> '16M', 25=> '32M', 26=> '64M', 27=> '128M', 28=> '256M', 29=> '512M', 30=> '1G' ); my %dec_checksum_type = ( 0 => 'none', 1 => 'crc32', 2 => 'crc32c' ); my %server_log_level = ( 0 => 'emerg', 1 => 'alert', 2 => 'crit', 3 => 'err', 4 => '???', 5 => 'notice', 6 => 'info', 7 => 'debug' ); foreach my $line (@$gather) { if ($line =~ /lg_block_sz.*=\s+(\d+)/) { verbose("HDFS Default Block Size: $lg_bln_sz{$1}"); } if ($line =~ /declared_checksum_type.*=\s+(\d+)/) { verbose("HDFS Default Checksum Type: $dec_checksum_type{$1}"); } if ($line =~ /log_level.*=\s+(\d+)/) { verbose("HDFS Server Log Level: $server_log_level{$1}"); } if ($line =~ /num_threads.*=\s+(\d+)/) { verbose("HDFS Server Threads: $1"); } } # Flag upgrade compatibility document for upgrades if (isOneFSUpgrade()) { addCustomerInformation('For information about OneFS compatibility with Hadoop, review the Using Hadoop with OneFS - Isilon Info Hub [https://www.dell.com/support/article/sln318813/using-hadoop-with-onefs-isilon-info-hub].'); } # Output Check result printResults(); } # End checkHDFS ### # checkHealth: Verifies cluster health status and node health status. $CHECKS{'checkHealth'}{'Description'} = "Verifies cluster health status and node health status."; sub checkHealth { return if wasCalled(); printCheck("Cluster Health Status"); gatherStatus(); gatherIsiStatPools(); gatherStat(); # Fail if cluster health is not OK if ($CLUSTER_INFO{'Health'} and $CLUSTER_INFO{'Health'} ne "OK") { fail("The cluster health is $CLUSTER_INFO{'Health'}"); } # Check each node for issues foreach my $node (sort {$a <=> $b} keys %NODE_INFO) { next if !$NODE_INFO{$node}{'Health'}; if ($NODE_INFO{$node}{'Health'} =~ /SMARTFAIL/) { # Fail if cluster is reporting SMARTFAIL fail("Node $node is reporting SMARTFAIL"); } else { if ($NODE_INFO{$node}{'Health'} =~ /.A../) { # Fail if cluster is reporting ATTENTION fail("Node $node is reporting ATTENTION"); } if ($NODE_INFO{$node}{'Health'} =~ /D.../) { # Fail if cluster is reporting DOWN fail("Node $node is reporting DOWN"); } if ($NODE_INFO{$node}{'Health'} =~ /..S./) { # Fail if cluster is reporting SMARTFAIL fail("Node $node is reporting SMARTFAIL"); } if ($NODE_INFO{$node}{'Health'} =~ /...R/) { # Fail if cluster is reporting READ-ONLY fail("Node $node is reporting READ-ONLY"); } } } info("Refer to KB 335100 (https://www.dell.com/support/kbdoc/335100) for details.") if isResults(&WARN); printResults(); } # End checkHealth $CHECKS{'checkIDI'}{'Description'} = "Checks for IDI errors in the past 90 days"; sub checkIDI { return if wasCalled(); printCheck("IDI Errors"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array cat /var/log/idi.log 2>&1]); } else { $gather = gatherCluster(qq[2>&1 grep -H . */varlog.tar/log/idi.log 2>&1]); } # Analyze information for issues. foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } my $idi_error_count = 0; foreach my $line (@{$gather->{$lnn}}) { # Gather timestamp of IDI error next unless ($line =~ /^\s*(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/); my $timestamp = timelocal($6, $5, $4, $3, $2 - 1, $1); # Ignore if older than 90 days next if $RPS{'Runtime'} - $timestamp > 7776000; # Gather IDI errors if ($line =~ /kernel:.*?(ENOTC\|Stack dump|idi_shallow_verify_failure|CRC|bhv|mmb|ffb|DSR failure)/) { $idi_error_count++; debug("CheckIDI: $line") } } # Review gathered IDI error information if ($idi_error_count > 0) { critical("Detected $idi_error_count IDI errors on node $lnn in the last 90 days"); } } # Output Check result printResults(); } # End checkIDI ### # checkISCSI: Checks for iSCSI LUNs being configured in /ifs/.ifsvar/iscsi/iscsi.conf $CHECKS{'checkISCSI'}{'Description'} = "Checks for iSCSI LUNs being configured in /ifs/.ifsvar/iscsi/iscsi.conf (OneFS prior to 8.x only)"; sub checkISCSI { if (wasCalled()) { return } if (isCurrentOneFS('>=8')) { return unsupported('Supported in OneFS 7 and earlier'); } if (!isDestinationOneFS('>=8')) { return unsupported('Supported in when target_version parameter is set'. ' to OneFS 8 or later'); } checkLicense(); printCheck("iSCSI LUNs"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 grep -n ^LUN /ifs/.ifsvar/iscsi/iscsi.conf"); } else { $gather = gatherLocal("2>&1 grep -n ^LUN local/ifsvar_iscsi.tar/iscsi/iscsi.conf"); } # Analyze information for issues. my @iscsiluns = (); foreach my $line (@$gather) { next unless $line =~ /^\d+:LUN/; next if $line =~ / -H/; push(@iscsiluns, $line); } foreach my $iscsilun (@iscsiluns) { fail("found active iSCSI LUN in line $iscsilun"); } info("The iSCSI module is removed in OneFS 8.x and later. To avoid issues with data associated with iSCSI LUNs following an upgrade, the LUNs must be inactive or removed prior to completing the upgrade checks. For details see https://www.dell.com/support/kbdoc/494931") if isResults(&FAIL); # Output Check result printResults(); } # End checkISCSI $CHECKS{'checkKB000181818'}{'Description'} = 'Checks for criteria of KB 000181818'; sub checkKB000181818 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2')) { return unsupported('Supported in OneFS 8 and OneFS 8.1'); } printCheck('KB 000181818'); # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array]. q[ 'ps auxw | grep -v grep | grep -c isi.hw.fw.proxy.Proxy']; my $CMD_LOGS = q[2>&1 grep -H -a -c isi.hw.fw.proxy.Proxy */ps]; my $MAX_PIDS = 4; my $SPF_PIDS = 'The following nodes have more than %s'. ' isi.hw.fw.proxy.Proxy processes running on them: %s'; my $MSG_KB = 'For additional information see KB 000181818'. ' [https://www.dell.com/support/kbdoc/en-us/000181818/stale-isi-hw-fw-proxy-proxy-processes-can-lead-to-clusterwide-deadlock]'; # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my %nodes = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach (@{$gather->{$lnn}}) { if (/^\s*(\d+)\s*$/ and $1 > $MAX_PIDS) { $nodes{$lnn} = $1 } } } # Flag if (%nodes) { fail(sprintf($SPF_PIDS, $MAX_PIDS, compressRange(keys %nodes))); info($MSG_KB); } # Output printResults(); } # End checkKB000181818 $CHECKS{'checkKB000192800'}{'Description'} = 'Checks for critera of KB 000192800'; sub checkKB000192800 { if (wasCalled()) { return } # Current OneFS version requirements if (isCurrentOneFS('>=9.2.1.5') or (isCurrentOneFS(qr/^9\.1\.0/) and isCurrentOneFS('>=9.1.0.12'))) { return unsupported ('Supported on OneFS versions prior to 9.1.0.12, 9.2.1.5, or 9.3.0.0'); } # Destination OneFS version requirements if (!isDestinationOneFS() or !isOneFSUpgrade()) { return unsupported ('Supported with target_version set to a OneFS version'); } if (isDestinationOneFS('<9.1.0.12') or (isDestinationOneFS(qr/^9\.2\.1/) and isDestinationOneFS('<9.2.1.5'))) { return unsupported ('Supported when target_version is newer than 9.1.0.12, 9.2.1.5, or 9.3.0.0'); } # Setup printCheck('KB 000192800'); my $CMD_GREP = q[egrep '^driveconfigs\.[0-9]+( |\.interface_type)']; my $FILE_LIVE = q[/ifs/.ifsvar/modules/hardware/drive_config.gc]; my $FILE_LOG = q[local/ifsvar_modules.tar/modules/hardware/drive_config.gc]; my $CMD_LIVE = qq[2>&1 $CMD_GREP $FILE_LIVE]; my $CMD_LOGS = qq[2>&1 $CMD_GREP $FILE_LOG]; my $RE_NUM = qr/^driveconfigs\.(\d+)/; my $RE_INTERFACE_TYPE = qr/interface_type/; my $RE_NONE = qr/none/; my $MSG_RISK = 'Drive config files put this cluster at risk for encountering the'. ' issues outlined in KB 000192800 during an upgrade to OneFS'. ' 9.1.0.12, 9.2.1.5, 9.3.0.0 or newer releases'; my $MSG_WORKAROUND = 'Prior to starting an OneFS upgrade, perform the proactive measures'. ' in KB 000192800'; my $MSG_KB = 'For more details see KB 000192800'. ' (https://www.dell.com/support/kbdoc/000192800)'; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $at_risk = 0; my $previous = undef; my $current = undef; my $found = 0; foreach my $line (@$gather) { if ($at_risk) { last } if ($line !~ $RE_NUM) { next } $current = $1; if (defined $current and defined $previous and $current ne $previous) { if (!$found) { $at_risk = 1 } $found = 0; } if ($line =~ $RE_INTERFACE_TYPE) { $found = 1 } if ($line =~ $RE_NONE) { $at_risk = 1 } $previous = $current; } if (defined $current and !$found) { $at_risk = 1 } # Flag if ($at_risk) { critical($MSG_RISK); critical($MSG_WORKAROUND); info($MSG_KB); } # Output printResults(); } # End checkKB000192800 ### # checkKB496582: Checks for auth rules issues detailed in KB496582. $CHECKS{'checkKB496582'}{'Description'} = "Checks for auth rules issues detailed in KB496582."; sub checkKB496582 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.0.0.4')) { return unsupported('Supported in OneFS 8.0.0.3 and earlier'); } if (!isDestinationOneFS('>=8.0.0.4')) { return unsupported('Supported when target_version parameter is OneFS'. ' 8.0.0.4 or later'); } printCheck("KB 496582"); my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi_gconfig | grep MappingRules"); } elsif (-e "local/isi_gconfig") { $gather = gatherLocal("2>&1 grep MappingRules local/isi_gconfig"); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { $gather = gatherLocal("2>&1 grep MappingRules local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } my %rules = (); foreach my $line (@$gather) { next unless ($line =~ /Zones\.([^\.]+)\.MappingRules [^\[]+\[ (.+) \]/); $rules{$1} = $2; debug("checkKB496582: Zone: $1 Rules: $rules{$1}\n"); } my $KB496582 = 0; foreach my $zone (keys %rules) { foreach my $rule (parseRules($rules{$zone})) { $KB496582 = 1 if ($rule->{'options'} and $rule->{'options'} =~ /group/ and $rule->{'options'} !~ /groups/); } } if ($KB496582) { warning("Customer should review KB 496582 prior to the OneFS upgrade, a User Mapping Rule was found that may be impacted by the change of behavior in OneFS 8.0.0.4."); addCustomerInformation("Review KB 496582 prior to the OneFS upgrade, a User Mapping Rule was found that may be impacted by the change of behavior in OneFS 8.0.0.4."); } printResults(); } # End checkKB496582 $CHECKS{'checkKB501267'}{'Description'} = 'Checks for the criteria of KB 501267'; sub checkKB501267 { if (wasCalled()) { return } # Setup printCheck('KB 501267'); my $CMD_NODE = 'sysctl efs.bam.disk_pool_db|grep generation'; my $CMD_LIVE = "2>&1 ${SUDO} isi_for_array -s '${CMD_NODE}'"; my $CMD_LOGS = '2>&1 grep -H generation */efs.bam.disk_pool_db'; my $MSG_MULTI = 'Multiple generations observed in Disk Pools DB'; my $SPF_GEN = 'generation=%s on nodes: %s'; my $MSG_KB = 'For details see KB 501267'. ' [https://www.dell.com/support/kbdoc/en-us/501267 - Unprovisioned'. ' drives reported by some nodes in cluster, other nodes report'. ' Unprovisioned drives: none]'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Analyze my %generations = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/generation=(\d+)/) { push(@{$generations{$1}}, $lnn); } } } # Flag if (scalar(keys %generations) > 1) { warning($MSG_MULTI); foreach (sort keys %generations) { warning(sprintf($SPF_GEN, $_, compressRange(@{$generations{$_}}))); } warning($MSG_KB); } # Output printResults(); } # End checkKB501267 $CHECKS{'checkKB507031'}{'Description'} = "Checks for criteria outlined in KB507031"; sub checkKB507031 { if (wasCalled()) { return } return unsupported("Does not apply in OneFS 8.2 and later") if isCurrentOneFS('>=8.2'); printCheck("KB507031"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array grep -e 'Match' -e 'Ciphers' /etc/mcp/templates/sshd_config]); } else { $gather = gatherCluster(qq[2>&1 grep -H -e 'Match' -e 'Ciphers' */etc.tar/etc/mcp/templates/sshd_config]); } # Analyze information for issues. my @ciphers_after_mb = (); my @mb_without_ciphers = (); my @commented_mb_with_ciphers = (); foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; my $has_mb = 0; my $has_ciphers = 0; my $has_commented_mb = 0; foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /^\s*Match/) { $has_mb = 1; } if ($line =~ /^\s*Ciphers/) { $has_ciphers = 1; if ($has_mb) { my @results = grep {$_ eq $lnn} @ciphers_after_mb; push(@ciphers_after_mb, $lnn) unless @results; } } if ($line =~ /^\s*#MB/) { $has_commented_mb = 1; } } push(@mb_without_ciphers, $lnn) if $has_mb and !$has_ciphers; push(@commented_mb_with_ciphers, $lnn) if $has_ciphers and $has_commented_mb; } if (@mb_without_ciphers and isDestinationOneFS()) { critical("Follow the 'Preventative Workaround for OneFS upgrades' steps as outlined in KB 507031 when performing the OneFS upgrade to $RPS{'Target'}."); } if (@ciphers_after_mb) { fail("Directive 'Ciphers' is not allowed within a Match block, review KB 507031 for resolution."); } if (@commented_mb_with_ciphers) { fail("Follow the 'Preventative Workaround for OneFS upgrades' post-upgrade steps as outlined in KB 507031."); } # Output Check result printResults(); } # End checkKB507031 $CHECKS{'checkKB519388'}{'Description'} = "Pre-upgrade check for issues outlined in KB519388"; sub checkKB519388 { if (wasCalled()) { return } if (isCurrentOneFS('<8') or isCurrentOneFS('>=8.1.2')) { return unsupported('Supported in OneFS 8 releases prior to 8.1.2.0'); } if (isDestinationOneFS('>=8.1.2')) { return unsupported('Supported when target_version is a OneFS release'. ' prior to 8.1.2.0'); } printCheck("KB519388"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'sysctl efs.snapshot.default_sub_visible']); } else { $gather = gatherCluster(qq[2>&1 grep -H default_sub_visible */efs.snapshot]); } # Analyze information for issues. verboseHeader("KB519388"); verbose(sprintf("%-10s %-10s", "LNN", "efs.snapshot.default_sub_visible")); verbose(sprintf("%-10s %-10s", "-"x10, "-"x10)); my $default_sub_visible = 0; foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /efs\.snapshot\.default_sub_visible:\s+(\d+)$/; $default_sub_visible = 1 if $1; verbose(sprintf("%-10s %-10s", $lnn, $1)); } } critical("The sysctl setting efs.snapshot.default_sub_visible is currently enabled", "During an upgrade from OneFS $CLUSTER_INFO{'OneFS'} to $RPS{'Target'} the sysctl setting efs.snapshot.default_sub_visible should be disabled, see KB519388 details on how to disable this setting.", "Take care to re-enable this setting once the upgrade has completed.") if $default_sub_visible; # Output Check result printResults(); } # End checkKB519388 $CHECKS{'checkKB519423'}{'Description'} = "Checks if the cluster config files are in a mixed mode"; sub checkKB519423 { if (wasCalled()) { return } return unsupported("Supported in OneFS 8 and later") unless isCurrentOneFS('>=8'); return unsupported("Unsupported on Compliance Mode enabled clusters") if isCompliance() and isLive(); printCheck("KB519423"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal (q[for i in /ifs/.ifsvar/modules/tardis/namespaces/*.sqlite; ]. q[do echo $i; sqlite3 $i "select * from revision_table"; ]. q[done 2>&1]); } else { $gather = gatherLocal (q[for i in local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/*.sqlite; ]. q[do echo $i; sqlite3 $i "select * from revision_table"; ]. q[done 2>&1]); } # Analyze information for issues. my $tardis_file = ""; my %revision_count = (); verboseHeader("KB519423"); foreach (@$gather) { if (/.sqlite$/) { verbose("No revision data") if $tardis_file and $revision_count{$tardis_file} == 0; $tardis_file = $_; $revision_count{$tardis_file} = 0; verbose(""); verbose("-"x20); verbose($tardis_file); verbose("-"x20); } else { verbose($_); } next unless $tardis_file; $revision_count{$tardis_file}++ if /^\d+\|\d+$/; } verbose("No revision data") if $tardis_file and $revision_count{$tardis_file} == 0; # Check for multiple revisions my @multiple_revisions = (); my @no_revisions = (); foreach my $file (keys %revision_count) { push(@multiple_revisions, $file) if $revision_count{$file} > 1; push(@no_revisions, $file) if $revision_count{$file} == 0; } # Flag issues detected if (@multiple_revisions) { fail("There are cluster config files with mixed revision data, this should only occur during an OneFS upgrade."); fail("If there is no OneFS upgrade currently in process, escalate to Isilon Support and reference KB 519423"); } if (@no_revisions) { fail("There are cluster config files with no revision data."); fail("Escalate to Isilon Support and reference KB 519423"); } # Output Check result printResults(); } # End checkKB519423 $CHECKS{'checkKB521778'}{'Description'} = "Checks for criteria outlined in KB 521778"; sub checkKB521778 { if (wasCalled()) { return } # Skip unless running or upgrading to impacted versions return unsupported("Only applies to OneFS 8.0.0.5, OneFS 8.0.0.6, OneFS 8.1.0.2, and OneFS 8.1.0.3") unless ($RPS{'Target'} and $RPS{'Target'} =~ /^(8\.0\.0\.[56]|8\.1\.0\.[23])$/) or (!$RPS{'Target'} and isCurrentOneFS(qr/^(8\.0\.0\.[56]|8\.1\.0\.[23])$/)); # Skip unless licensed for SmartQuotas and SyncIQ return unsupported("Only applies when using both SmartQuotas and SyncIQ") unless hasLicense("SmartQuotas") and hasLicense("SyncIQ"); # Skip unless a SyncIQ target gatherSyncIQ(); return unsupported("Only applies to targets of SyncIQ") unless $CLUSTER_INFO{'SyncIQ'} and $CLUSTER_INFO{'SyncIQ'}{'Target'}; printCheck("KB521778"); gatherPatches(); warning("This cluster is licensed for SmartQuotas and is a target of SyncIQ. Incoming SyncIQ may fail if there are more than 100 quotas in sub-directories below a SyncIQ target path. See KB 521778 for details on this known issue."); $RPS{'KB521778'} = 1; if ($RPS{'Target'}) { if ($RPS{'Target'} =~ /^(8\.0\.0\.5)$/) { critical("Do NOT perform the SyncIQ Rollup Patch 224399 after performing an upgrade to OneFS $RPS{'Target'}."); } elsif ($RPS{'Target'} =~ /^(8\.0\.0\.6)$/) { critical("Do NOT perform the SyncIQ Rollup Patch 215690 or Patch 221638 after performing an upgrade to OneFS $RPS{'Target'}."); } elsif ($RPS{'Target'} =~ /^(8\.1\.0\.2)$/) { critical("Do NOT perform the SyncIQ Rollup Patch 222718 after performing an upgrade to OneFS $RPS{'Target'}."); } elsif ($RPS{'Target'} =~ /^(8\.1\.0\.3)$/) { fail("An upgrade to OneFS $RPS{'Target'} should NOT be performed."); } } if (hasPatch("patch-(224399|215690|221638|222718)") or isCurrentOneFS(qr/^(8\.1\.0\.3)$/)) { if ($RPS{'Target'}) { warning("Cluster current OneFS and patch level put it at risk for KB 521778."); } else { fail("Cluster current OneFS and patch level put it at risk for KB 521778."); } } elsif (isCurrentOneFS('8.0.0.5')) { warning("Do NOT perform the SyncIQ Rollup Patch 224399"); } elsif (isCurrentOneFS('8.0.0.6')) { warning("Do NOT perform the SyncIQ Rollup Patch 215690 or Patch 221638"); } elsif (isCurrentOneFS('8.1.0.2')) { warning("Do NOT perform the SyncIQ Rollup Patch 222718"); } # Output Check result printResults(); } # End checkKB521778 $CHECKS{'checkKB524082'}{'Description'} = 'Checks if the cluster is enabled'. ' for HTTP clients and flags a compatibility issue caused by a change in'. ' Apache versions.'; sub checkKB524082 { if (wasCalled()) { return } # Skip if not upgrading to a version with Apache 2.4 if (!$RPS{'Target'}) { return } if (compareVersions($RPS{'Target'}, '8.0.0.6') <= 0 or isDestinationOneFS(qr/^8\.0\.1\.[01]/) or isDestinationOneFS(qr/^8\.1\.0\.[012]/)) { return; } # Skip if running a version with Apache 2.4 if (isCurrentOneFS('8.0.0.7') or isCurrentOneFS('8.1.0.3') or isCurrentOneFS('8.1.0.4') or isCurrentOneFS('>=8.1.2')) { return; } # Setup printCheck('KB 524082'); my $CMD_LIVE6 = qq[2>&1 grep "httpdata2" /etc/mcp/override/apache2.xml]; my $CMD_LOGS6 = q[2>&1 grep "httpdata2"]. q[ */etc.tar/etc/mcp/override/apache2.xml]. q[ local/ifsvar_etc.tar/etc/mcp/override/apache2.xml]; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_gconfig -t http-config]; my $CMD_LOGS = q[2>&1 cat]. q[ local/ifsvar_modules.tar/modules/http/http_config.gc]; my $MSG = "During the upgrade from OneFS $CLUSTER_INFO{'OneFS'} to OneFS". " $RPS{'Target'}, Apache is upgraded from Apache 2.2 to Apache 2.4.". ' If .htaccess files are utilized, this can result in HTTP 500 errors'. ' following and during the upgrade due to invalid existing Apache 2.2'. ' syntax. Please review KB 524082'. ' [https://www.dell.com/support/kbdoc/524082 - Upgrading from Apache '. ' 2.2 to Apache 2.4 can result in HTTP 500 errors due to invalid 2.2 '. ' syntax in .htaccess files] prior to the OneFS upgrade to determine '. ' whether this may be impactful to your workflow.'; # Gather my $gather = (); if (isCurrentOneFS('<7')) { if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE6); } else { $gather = gatherLocal($CMD_LOGS6); } } else { if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } } # Analyze my $enabled_config = 0; foreach my $line (@$gather) { if ($line =~ /^redirect.*false/ or $line =~ /httpdata2.*>enabled=8')) { return unsupported('Supported in OneFS 7 and earlier'); } if (!$RPS{'Target'} or compareVersions($RPS{'Target'}, '8') < 0) { return unsupported('Supported in upgrades to OneFS 8 or later'); } # Setup printCheck('KB 527312'); my $CMD_LIVE = q[2>&1 cat /ifs/.ifsvar/modules/smartpools]. q[/drive_purposing.db]; my $CMD_LOGS = q[2>&1 cat local/ifsvar_modules.tar/modules/smartpools]. q[/drive_purposing.db]; my $SPF_UPGRADE = 'Cluster may be affected by KB 527312 during an upgrade'. ' from %s to %s.'; my $MSG_AFFECTED = 'The malformed drive_purposing.db file should be'. ' corrected prior to performing the OneFS upgrade.'; my $MSG_KB = 'For details see KB 527312'. ' [https://www.dell.com/support/kbdoc/527312 - {ISILON} When upgrading'. ' from OneFS 7.X to 8.X, upgrade scripts may not complete if there is'. ' an incorrect entry in drive_purposing.db'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze my $affected = 0; foreach my $line (@$gather) { # If there are any empty lists they may be affected if ($line =~ /\[\s*\]/) { $affected = 1 } # Once found affected, drop out of loop if ($affected) { last } } # Flag if ($affected) { fail(sprintf($SPF_UPGRADE, $CLUSTER_INFO{'OneFS'}, $RPS{'Target'})); fail($MSG_AFFECTED); info($MSG_KB); } # Output printResults(); } $CHECKS{'checkKB530050'}{'Description'} = 'Check for criteria of KB 530050'; sub checkKB530050 { if (wasCalled()) { return } if (isCurrentOneFS('<7.2.1')) { return unsupported('Supported in OneFS 7.2.1 and later'); } # Setup printCheck('KB 530050'); my $CMD_NODE = q[ls /etc/hardening[-_]metadata*]; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array '$CMD_NODE']; my $CMD_LOGS = q[2>&1 grep -H . -m 1 */etc.tar/etc/hardening[-_]metadata*]; my $SPF_NODES = 'The following nodes may be impacted by KB 530050 during a'. ' OneFS upgrade: %s'; my $MSG_KB = 'For details see KB 530050'. ' [https://www.dell.com/support/kbdoc/530050 - Isilon: Unable to'. ' determine security hardening state on the cluster.'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Analyze my @nodes = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { if ($RPS{'Live'}) { missing($lnn); } else { push(@nodes, $lnn); } next; } foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /no matches found/) { push(@nodes, $lnn); last; } } } # Flag if (%nodes_with_issues) { fail(sprintf($SPF_NODES, compressRange(@nodes))); info($MSG_KB); } # Output printResults(); } $CHECKS{'checkKB540000'}{'Description'} = 'Checks for criteria of KB 540000'; sub checkKB540000 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2')) { return unsupported('Supported in OneFS 8.1 and earlier'); } # Setup printCheck('KB 540000'); my $GCONFIG = 'fsa.max_count'; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_gconfig $GCONFIG]; my $CMD_LOGS = qq[2>&1 grep -h $GCONFIG local/isi_gconfig]. q[ local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc]; my $SPF_FSA = 'The maximum number of FSA results that can be saved in'. ' OneFS 8.2 and later is 99. The current value of the fsa.max_count'. ' is %s and would be changed to a default value of 8 after an upgrade'. ' to OneFS 8.2.0 or later if left unchanged.'; my $MSG_KB = 'For additional details review KB 540000'. ' [https://www.dell.com/support/kbdoc/540000 - Isilon OneFS: FSA'. ' results deleted after upgrading OneFS]'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Parse my ($key, $value) = parseGconfigItem(@$gather); # Flag if ($key eq 'fsa.max_count' and ($value < 1 or $value > 99)) { my $msg = sprintf($SPF_FSA, $value); warning($msg); if ($RPS{'Target'} and compareVersions($RPS{'Target'}, '8.2') >= 0) { addCustomerInformation("$msg $MSG_KB"); } info($MSG_KB); } # Output printResults(); } $CHECKS{'checkKB540071'}{'Description'} = 'Checks for existence of files under'. ' /var/fw/fwpkg when no IsiFw package is installed'; sub checkKB540071 { if (wasCalled()) { return } if (hasPatch('^IsiFw')) { return unsupported('Supported when no IsiFw package is installed'); } # Setup printCheck('KB 540071'); my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'du -hs /var/fw/fwpkg']; my $CMD_LOGS = q[2>&1 grep -H 'fwpkg$' */du]; my $SPF_NODES = 'The following nodes are at risk for KB 540071: %s'; my $MSG_KB = 'For additional details review KB 540071'. ' [https://www.dell.com/support/kbdoc/540071 - Isilon: /var/fw/fwpkg'. ' exists but no Isilon Firmware package installed]'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } else { $gather = gatherCluster($CMD_LOGS); } # Analyze my @nodes_at_risk = (); foreach my $lnn (@LNNS) { foreach (@{$gather->{$lnn}}) { if (/^\s*\S+\s+\/var\/fw\/fwpkg$/) { push(@nodes_at_risk, $lnn); } } } # Flag if (@nodes_at_risk) { critical(sprintf($SPF_NODES, compressRange(@nodes_at_risk))); info($MSG_KB); } # Output printResults(); } $CHECKS{'checkKB540872'}{'Description'} = 'Checks if the cluster may encounter'. ' KB 540872 during an upgrade from OneFS 8.2 releases'; sub checkKB540872 { if (wasCalled()) { return } if (!isCurrentOneFS('8.2.0.0') and !isCurrentOneFS('8.2.1.0') and !isCurrentOneFS('8.2.2.0')) { return unsupported('Supported in OneFS 8.2 releases'); } # Setup printCheck('KB 540872'); my $GCONFIG = 'smartpools.diskpools.suggested_protection_enabled'; my $CMD_LIVE_GCONFIG = qq[2>&1 ${SUDO} isi_gconfig $GCONFIG]; my $CMD_LOG_GCONFIG = qq[2>&1 grep $GCONFIG local/isi_gconfig]; my $CMD_LIVE_DRIVEDB = q[2>&1 cat /ifs/.ifsvar/modules/smartpools/drive_purposing.db]; my $CMD_LOG_DRIVEDB = q[2>&1 cat local/ifsvar_modules.tar/modules]. q[/smartpools/drive_purposing.db]; my $SPF_MTTDL_MISSING = 'The following equivalence entries in the drive'. ' purposing database are missing the "mttdl" key and puts the cluster'. ' at risk for encountering KB 540872 during an OneFS upgrade: %s'; my $SPF_NO_NODES = 'The following equivalence entries in the drive'. ' purposing database do not have any active node device ids in'. ' them and puts the cluster at risk for encountering KB 540872'. ' during an upgrade to OneFS 8.2.2: %s'; my $MSG_PROT = 'Suggested Protection is not enabled'; my $MSG_KB540872 = 'For more details see KB 540872 '. ' [https://www.dell.com/support/kbdoc/540872 - Isilon: OneFS'. ' upgrade may fail during'. ' upgrade-post-reboot-initial/smartpools_db_consistency_check]'; # Gather suggested_protection_enabled my $suggested_protection_enabled = 1; my $gather_gconfig = (); if ($RPS{'Live'}) { $gather_gconfig = gatherLocal($CMD_LIVE_GCONFIG); } else { $gather_gconfig = gatherLocal($CMD_LOG_GCONFIG); } # Analyze suggested_protection_enabled foreach (@$gather_gconfig) { my ($key, $value) = parseGconfigItem($_); if ($key =~ /$GCONFIG/ and $value =~ /false/) { $suggested_protection_enabled = 0; } } # Gather drive_purposing.db my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE_DRIVEDB); } else { $gather = gatherLocal($CMD_LOG_DRIVEDB); } # Analyze drive_purposing.db my $db = join('', @$gather); my $ee_missing_mttdl = ''; my $ee_no_nodes = ''; eval { require JSON::PP; import JSON::PP qw( decode_json ); my $href_json = decode_json($db); my %ee = %{$href_json->{equivalence_entries}}; my @mttdl_missing = (); my @no_nodes = (); foreach my $entry (keys %ee) { if (!defined $ee{$entry}{mttdl}) { debug("Equivalence entry $entry is missing mttdl"); push(@mttdl_missing, $entry); } if (!defined $ee{$entry}{node_types}) { debug("Equivalence entry $entry is missing node_types"); next; } TYPE: foreach my $type (@{$ee{$entry}{node_types}}) { if (ref $type ne 'ARRAY') { next } foreach my $element (@$type) { if (ref $element ne 'ARRAY') { next } foreach my $devid (@$element) { if (defined $LNN_FROM_DEVID{$devid}) { next TYPE } } } push(@no_nodes, $entry); } } $ee_missing_mttdl = join(', ', sort {$a <=> $b} @mttdl_missing); $ee_no_nodes = join(',', sort {$a <=> $b} @no_nodes); }; # Flag if ($ee_missing_mttdl) { fail(sprintf($SPF_MTTDL_MISSING, $ee_missing_mttdl)); } if ($ee_no_nodes) { if (isDestinationOneFS('8.2.2.0')) { fail(sprintf($SPF_NO_NODES, $ee_no_nodes)); } else { warning(sprintf($SPF_NO_NODES, $ee_no_nodes)); } } if (!$suggested_protection_enabled) { info($MSG_PROT) } # Output if (isResults(&INFO)) { info($MSG_KB540872) } printResults(); } $CHECKS{'checkKB544401'}{'Description'} = 'Check for critera of KB 544401'; sub checkKB544401 { if (wasCalled()) { return } if (!isCurrentOneFS('8.2.0.0')) { return unsupported('Supported in OneFS 8.2.0'); } # Setup printCheck('KB 544401'); gatherSRS(); my $SPF_NODES = 'The following nodes are not considered eligible for SRS'. ' connectivity and may get stuck during shutdown: %s'; my $MSG_SRS = 'To avoid this issue, prior to performing any scheduled'. ' maintenance tasks which would reboot the nodes, disable the'. ' isi_esrs_d service by running: isi services -a isi_esrs_d disable.'. ' Once the maintenance tasks have concluded, re-enable the isi_esrs_d'. ' service by running: isi services -a isi_esrs_d enable'; my $MSG_KB = 'For more details see KB 544401'. ' [https://www.dell.com/support/kbdoc/544401 - During upgrade of OneFS'. ' from 8.1.x or 8.2.0 to later versions, NANON nodes or nodes without'. ' ESRS gateway access may get stuck during shutdown]'; my $RE_PATCH = '^8\.2\.0_.*[-_](?:255568|257594|259123|261103|261104'. '|262892|262893|264781|264782|266617|266990|268444|269435|270016'. '|271507|273332|273851|64|172|320)$'; # Analyze my @nodes = (); foreach my $lnn (@LNNS) { my ($match) = grep {$_ eq $lnn} @{$CLUSTER_INFO{'SRS'}{'Nodes'}}; if (!$match) { push(@nodes, $lnn) } } # Flag if (@nodes and isServiceEnabled('isi_esrs_d') and !hasPatch($RE_PATCH)) { if ($RPS{'JSON'}) { warning(sprintf($SPF_NODES, compressRange(@nodes))); warning($MSG_SRS); } else { critical(sprintf($SPF_NODES, compressRange(@nodes))); critical($MSG_SRS); } info($MSG_KB); } # Output printResults(); } $CHECKS{'checkKB544854'}{'Description'} = 'Check for criteria of KB 544854'; sub checkKB544854 { if (wasCalled()) { return } if (isCurrentOneFS('<8.2')) { return unsupported('Supported in OneFS 8.2 and later'); } printCheck('KB 544854'); gatherArrayXML(); # Setup my $MSG_FILESERIAL = 'Multiple versions of the array config file'. ' [/etc/ifs/array.xml] detected. To resolve this issue, restart the'. ' isi_array_d service by running the following command: isi_for_array'. ' killall isi_array_d'; my $MSG_KB = 'For more details see KB 544854'. ' [https://www.dell.com/support/kbdoc/544854 - Isilon: After'. ' upgrade commit, array.xml is not consistent across all nodes.]'; my $SPF = '%5s %10s'; # Analyze my %fileserials = (); verboseHeader('/etc/ifs/array.xml fileserials'); verbose(sprintf($SPF, 'LNN', 'fileserial')); verbose(sprintf($SPF, '-'x5, '-'x10)); foreach my $lnn (@LNNS) { if (!$NODE_INFO{$lnn}{'fileserial'}) { next } verbose(sprintf($SPF, $lnn, $NODE_INFO{$lnn}{'fileserial'})); push(@{$fileserials{$NODE_INFO{$lnn}{'fileserial'}}}, $lnn); } # Flag if (scalar(keys %fileserials) > 1) { fail($MSG_FILESERIAL); info($MSG_KB); } # Output printResults(); } ### # checkLicense: Checks licenses and gives direction based on licensed # features. InsigntIQ and vCenter licenses, gives information from # compatibility guide. iSCSI, instructs to only perform simultaneous # OneFS upgrades and that it is not supported in 8.0. $CHECKS{'checkLicense'}{'Description'} = "Checks licenses and gives direction based on licensed features. InsigntIQ and vCenter licenses, gives information from compatibility guide. iSCSI, instructs to only perform simultaneous OneFS upgrades and that it is not supported in 8.0."; sub checkLicense { return if wasCalled(); printCheck("Licenses"); gatherLicense(); gatherProductInfo(); if (hasLicense("iSCSI")) { if ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.0") >= 0) { fail("iSCSI is licensed. Support for iSCSI is deprecated as of OneFS 8."); } else { warning("iSCSI is licensed. If there are any iSCSI connections to the cluster, you can only perform a simultaneous upgrade. After all the initiator connections are stopped, set the iSCSI LUN to offline until the upgrade is complete." ); } } if ($RPS{'Target'} and hasLicense("vCenter")) { if (compareVersions($RPS{'Target'}, "7.2") >= 0 and isCurrentOneFS('<7.2')) { warning("Isilon for vCenter is licensed. OneFS 7.2 or later does not support Isilon for vCenter plugin."); } if (compareVersions($RPS{'Target'}, "7.0") >= 0 and isCurrentOneFS('<7')) { warning("Isilon for vCenter is licensed. OneFS 7.0 or later supports Isilon for vCenter version 1.3 or later."); } } # InsightIQ and OneFS Compatibilty my $SPF_IIQ = 'InsightIQ %s or later is required for OneFS %s support'; if (isDestinationOneFS() and (hasLicense('InsightIQ') or isCurrentOneFS('>=8.1'))) { if (isDestinationOneFS('>=9.2') and isCurrentOneFS('<9.2')) { warning(sprintf($SPF_IIQ, '4.1.4.60', isDestinationOneFS())); } elsif (isDestinationOneFS('>=8.2') and isCurrentOneFS('<8.2')) { warning(sprintf($SPF_IIQ, '4.1.3', isDestinationOneFS())); } elsif ((isDestinationOneFS('>=8.1.2') and isCurrentOneFS('<8.1.2')) or (isDestinationOneFS('>=8.1.1.1') and isCurrentOneFS('<8.1.1.1')) or (isDestinationOneFS('>=8.1.0.3') and isCurrentOneFS('<8.1.0.3')) or (isDestinationOneFS('>=8.0.0.7') and isCurrentOneFS('<8.0.0.7'))) { warning(sprintf($SPF_IIQ, '4.1.2', isDestinationOneFS())); } elsif (isDestinationOneFS('>=8') and isCurrentOneFS('<8')) { warning(sprintf($SPF_IIQ, '4.0', isDestinationOneFS())); } elsif (isDestinationOneFS('>=7.2') and isCurrentOneFS('<7.2')) { warning(sprintf($SPF_IIQ, '3.1', isDestinationOneFS())); } elsif (isDestinationOneFS('>=7') and isCurrentOneFS('<7')) { warning(sprintf($SPF_IIQ, '2.5', isDestinationOneFS())); } } # Flag major/minor/sub upgrades that do not have a valid signature if ($RPS{'Target'} and isCurrentOneFS('>=8.1') and not $CLUSTER_INFO{'Valid Signature'} and not isVirtual() and upgradeDigit($CLUSTER_INFO{'OneFS'}, $RPS{'Target'}) <= 3) { fail("License does not have a valid signature. The license must be updated prior to performing the upgrade from $CLUSTER_INFO{'OneFS'} to $RPS{'Target'}."); } # All information above here is in licenses checks should be sent to the customer addCustomerInformation($_) foreach (@Logger::WARN, @Logger::FAIL); # Flag is iSCSI is disabled but the iSCSI service is enabled as a warning if ((!$RPS{'Target'} or compareVersions($RPS{'Target'}, "8.0") < 0) and !hasLicense("iSCSI") and isServiceEnabled('isi_iscsi_d')) { warning("iSCSI license is $CLUSTER_INFO{'License'}{'iSCSI'}{'Status'} and the iSCSI target Daemon service(isi_iscsi_d) is enabled"); } # Flag expired licenses as a warning my @expired_licenses = (); foreach my $name (keys %{$CLUSTER_INFO{'License'}}) { if ($CLUSTER_INFO{'License'}{$name}{'Status'} and $CLUSTER_INFO{'License'}{$name}{'Status'} eq "Expired") { push(@expired_licenses, $name); } } warning("The following licenses have expired: ".join(", ", @expired_licenses)) if @expired_licenses; # Output Check result info("Refer to KB335095 (https://www.dell.com/support/kbdoc/335095) for details.") if isResults(&WARN); printResults($CLUSTER_INFO{'OneFS License ID'} or ""); } # End checkLicense $CHECKS{'checkLinMasterPadding'}{'Description'} = 'Checks the LIN master padding to be all zeros'; sub checkLinMasterPadding { if (wasCalled()) { return } if (!$RPS{'Live'}) { return unsupported('Only supported on live clusters'); } if (isCurrentOneFS('>=8.2') or isCurrentOneFS('<8')) { return unsupported('Only supported in OneFS 8.0 - 8.1'); } # Setup printCheck('LIN Master Zero Padding'); my $CMD_SYSCTL = q[sysctl efs.lin.super.cache]; my $GREP_LIN = q[grep -Eo "[1-9][0-9]*,[0-9]+,[0-9]+:8192"]; my $CMD_CPR = qq[2>/dev/null ${SUDO} isi_cpr -1rx bARG]; my $CMD_HEXDUMP = q[hexdump -s 0x285 -n 7537]; my $CMD_XARGS = qq[xargs -IARG sh -c "$CMD_CPR|$CMD_HEXDUMP"]; my $CMD_LIVE = qq[${SUDO} isi_for_array '$CMD_SYSCTL|$GREP_LIN|$CMD_XARGS']; my $MSG_NOT_ZERO_PADDING = 'Failed LIN master zero padding test'; my $MSG_KB = 'For more details review KB 544758'. ' [https://www.dell.com/support/kbdoc/en-us/544758 - Isilon: Node'. ' panics shortly after upgrading to OneFS 8.2]'; my @expected_output = ( '0000285 0000 0000 0000 0000 0000 0000 0000 0000', '*', '0001ff5', ); my $expected_size = scalar(@expected_output); # Gather my $gather = gatherCluster($CMD_LIVE); # Analyze my $has_zero_padding = 0; foreach my $lnn (@LNNS) { if ($has_zero_padding) { last } if (!$gather->{$lnn}) { next } my $gather_size = scalar(@{$gather->{$lnn}}); if ($gather_size % $expected_size) { next } my $lin_count = $gather_size / $expected_size; LIN: foreach my $l (0..$lin_count-1) { foreach my $e (0..$#expected_output) { my $g = $l * $expected_size + $e; $gather->{$lnn}[$g] =~ s/^\s+|\s+$//g; if ($gather->{$lnn}[$g] ne $expected_output[$e]) { next LIN } } $has_zero_padding = 1; last LIN; } } # Flag if (!$has_zero_padding) { if ($RPS{'Target'} and compareVersions($RPS{'Target'}, '8.2') >= 0) { fail($MSG_NOT_ZERO_PADDING); } else { warning($MSG_NOT_ZERO_PADDING); } info($MSG_KB); } # Output printResults(); } # End checkLinMasterPadding $CHECKS{'checkListenQueue'}{'Description'} = "Checks for listen queue overflows to be less than 50,000 per node"; sub checkListenQueue { return if wasCalled(); printCheck("Listen Queue Overflows"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array netstat -sn | grep 'listen queue overflow']); } else { $gather = gatherCluster(qq[2>&1 grep -H 'listen queue overflow' */netstat]); } # Analyze information for issues. my @listen_queue = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { next unless ($line =~ /(\d+)\s+listen queue overflow/); verbose("Listen Queue Overflows: $1"); push(@listen_queue, $lnn) if ($1 > 50000); } } warning("Listen Queue Overflows count over 50,000 on the following nodes: ".join(",", @listen_queue)) if @listen_queue; printResults(); } # End checkListenQueue $CHECKS{'checkLogLevel'}{'Description'} = 'Checks LWSM log levels for NFS, SMB, HDFS, and Authentication'; sub checkLogLevel { if (wasCalled()) { return } if (!isLive()) { return } if (isCompliance()) { return } if ($RPS{'JSON'}) { return } if (isCurrentOneFS('<8')) { return } # Setup printCheck('Log Level'); my %services = ( 'lsass' => 'authentication (lsass)', 'netlogon' => 'authentication (netlogon)', 'nfs' => 'NFS protocol', 'lwio' => 'SMB protocol', 'hdfs' => 'HDFS protocol', ); my $SPF_CMD = q[2>&1 isi_for_array '/usr/likewise/bin/lwsm get-log %s']; my $RE_DEBUG = qr/TRACE|DEBUG/i; my $SPF_DEBUG = 'Debug log levels found for %s on the following nodes: %s'; my $MSG_KB = 'For more information on log levels see KB 000008264 (https://www.dell.com/support/kbdoc/000008264/emc14002470-onefs-6-5-and-later-how-to-set-debug-logging-for-authentication-and-smb-services)'; # Gather my %gather = (); foreach my $service (keys %services) { $gather{$service} = gatherCluster(sprintf($SPF_CMD, $service)); } # Analyze my %debug_logging = (); foreach my $service (keys %gather) { foreach my $lnn (@LNNS) { if (!$gather{$service}{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather{$service}{$lnn}}) { if ($line =~ $RE_DEBUG) { push(@{$debug_logging{$service}}, $lnn); } } } } # Flag if (%debug_logging) { info($MSG_KB); foreach my $service (keys %debug_logging) { fail(sprintf($SPF_DEBUG, $services{$service}, compressRange(@{$debug_logging{$service}}))); } } # Output printResults(); } # End checkLogLevel $CHECKS{'checkLWIODLog'}{'Description'} = "Checks /var/log/lwiod.log for known errors occuring in the last 30 days"; sub checkLWIODLog { return if wasCalled(); printCheck("LWIOD Log"); # A list of the last 30 days my @date = (); for (my $i = 0; $i <= 30; $i++) { my ($day, $month, $year) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4,5]; $year += 1900; $month += 1; push(@date, sprintf("%04d-%02d-%02d", $year, $month, $day)); } # A list of the errors to gather my @detect = ('code:0xc0000205', 'GSS-API error calling gss_accept_sec_context:', 'Too many open files', 'Clock skew too great', 'LW_ERROR_CLOCK_SKEW', 'An invalid name was supplied', 'Configuration file does not specify default realm', 'Ticket not yet valid', 'Unspecified GSS failure. Minor code may provide more information', 'Illegal cross-realm ticket', '\.pst', '\.ovt', '\.pap', '\.ost', ); # A list of the errors to ignore my @ignore = ('LW_ERROR_ACCOUNT_DISABLED', 'LW_ERROR_INVALID_PASSWORD', 'LW_ERROR_ACCOUNT_LOCKED', 'LW_ERROR_ACCOUNT_EXPIRED', ); # Compile greps from the lists my $date_grep = join("|", @date); my $detect_grep = join("|", @detect); my $ignore_grep = join("|", @ignore); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'grep -E "^(${date_grep}).*(${detect_grep})" /var/log/lwiod.log | grep -v -E "(${ignore_grep})"']); } else { $gather = gatherCluster(qq[2>&1 grep -E "^(${date_grep}).*(${detect_grep})" */varlog.tar/log/lwiod.log | grep -v -E "(${ignore_grep})"]); } # Analyze information for issues. my $pst_errors = 0; my %error_counts = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless ($gather->{$lnn}); foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /\.(pst|ovt|pap|ost)/) { $pst_errors++; next; } elsif ($line =~ /($detect_grep)/) { $error_counts{$1}++; } } } foreach my $error (keys %error_counts) { warning("Found $error_counts{$error} '$error' errors"); } warning("Performance degrading workflow detected, see KB 334538 for details on performance impacts with PST files.") if $pst_errors; printResults(); } # End checkLWIODLog $CHECKS{'checkMaintenanceMode'}{'Description'} = 'Checks if the cluster is currently in maintenance mode'; sub checkMaintenanceMode { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return unsupported('Supported in OneFS 8 and later'); } printCheck('Maintenance Mode'); # Setup my $MSG_MAINT = q[The cluster is currently in maintenance mode. Review 'isi event]. q[ settings view' for the current maintenance mode settings. Use the]. q[ following command to take the cluster out of maintenance mode:]. q[ isi event settings modify --clear-maintenance-start]; my $MSG_MAINT_92 = q[The cluster is currently in maintenance mode. Use the following]. q[ command to take the cluster out of maintenance mode:]. q[ isi event maintenance modify disable]; my $CMD_LIVE = qq[2>&1 ${SUDO} isi --debug event settings view]; my $CMD_LOGS = q[2>&1 sqlite3 local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/celog.settings.sqlite 'select * from kv_table']; my $CMD_LIVE_92 = qq[2>&1 ${SUDO} isi event maintenance list --format json]; my $CMD_LOGS_92 = q[2>&1 sqlite3 local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/celog_maintenance.sqlite 'select * from kv_table']; if (isCurrentOneFS('>=9.2')) { # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE_92:$CMD_LOGS_92); # Analyze my $enabled = 0; foreach (@$gather) { if (/\"end\":\s*0\D/) { $enabled = 1 } } # Flag if ($enabled) { warning($MSG_MAINT_92) } } else { # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $start = 0; my $duration = 0; foreach (@$gather) { if (/\"duration\"\s*:\s*(\d+)\D/) { $duration = $1 } if (/\"start\"\s*:\s*(\d+)\D/) { $start = $1 } if (/maintenance_duration\|(\d+)\|/) { $duration = $1 } if (/maintenance_start\|(\d+)\|/) { $start = $1 } } # Flag if ($start + $duration > $RPS{'Runtime'}) { warning($MSG_MAINT) } } # Display printResults(); } # End checkMaintenanceMode $CHECKS{'checkMessagesLog'}{'Description'} = "Checks /var/log/messages.log for known errors occuring in the last 30 days"; sub checkMessagesLog { return if wasCalled(); printCheck("Messages Log"); # A list of the last 30 days my @date = (); for (my $i = 0; $i <= 30; $i++) { my ($day, $month, $year) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4,5]; $year += 1900; $month += 1; push(@date, sprintf("%04d-%02d-%02d", $year, $month, $day)); } # A list of the error types to gather my @detect = ("bus", "changed linkrate", "dsr", "dwr", "dws", "dwt", "ecc", "error", "fail", "free", "inquiry", "invalid hostname to sm_mon", "isi_flexnet_d", "isi_migrate", "lock", "mca", "memory", "nlm", "non-fatal", "onefs", "panic", "quota", "rbmn", "seconds in syslog", "stack", "stall", "terminated", "unrecovered", "upgrade", "userspace", "watchdog", ); my $awkbegin = "BEGIN{IGNORECASE=1}"; my $awkstring = ""; my $awkend = "END {"; foreach my $i (@detect) { my $key = $i; $key =~ s/[^a-zA-Z0-9]/_/g; $awkstring .= qq[/^[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]].*${i}/{${key}++; ${key}MR=\\\$1}]; $awkend .= qq[if(${key}){print \\\"${key}:\\\" ${key} \\\" MR:\\\" ${key}MR};]; } $awkend .= "}"; # A list of the errors to ignore my @ignore = ('last message repeated'); # Compile greps from the lists my $date_grep = join("|", @date); my $detect_grep = join("|", @detect); my $ignore_grep = join("|", @ignore); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'awk "]. qq[$awkbegin]. qq[(\\\$0 !~ /(${date_grep}).*\\.[01234]>/){next}]. qq[/(${ignore_grep})/{next}]. qq[$awkstring]. qq[$awkend]. qq[" /var/log/messages']); } else { $awkstring =~ s/\\//g; $awkend =~ s/\\//g; $gather = gatherCluster(qq[for i in */varlog.tar/log/messages; do 2>&1]. qq[ grep -H -E '^(${date_grep}).*\.[01234]>' \$i]. qq[|grep -v -E '(${ignore_grep})']. qq[|awk '$awkstring $awkend']. qq[|awk 'BEGIN{FS=":";OFS=":"}{LNN=\$3;\$3=\$2;\$2=\$1;\$1=LNN;print}']. qq[;done]); } # Analyze information for issues. my %error_counts = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless ($gather->{$lnn}); foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /^\s*([^:]+):(\d+) MR:(\S+)$/; $error_counts{$lnn}{$1}{'Count'} = $2; $error_counts{$lnn}{$1}{'Most Recent'} = $3; } } foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $error_counts{$lnn}; foreach my $error (keys %{$error_counts{$lnn}}) { warning("Node $lnn: Found $error_counts{$lnn}{$error}{'Count'} '$error' errors. Most recent log entry of this type was: $error_counts{$lnn}{$error}{'Most Recent'}"); } } printResults(); } # End checkMessagesLog $CHECKS{'checkNDMP'}{'Description'} = 'Checks for running NDMP sessions'; sub checkNDMP { if (wasCalled()) { return } printCheck('NDMP'); my $CMD_SESSIONS_7 = qq[2>&1 ${SUDO} isi ndmp list]; my $CMD_SESSIONS_8 = qq[2>&1 ${SUDO} isi ndmp sessions list]; my $CMD_LIVE_TRACE = q[2>&1 find /ifs/.ifsvar/modules/ndmp/trace/ -mtime -365 -type f -name '*.trace' | wc -l]; my $CMD_LOGS_TRACE = q[2>&1 find ARG/local/ifsvar_modules.tar/modules/ndmp/trace/ -mtime -365 -type f -name '*.trace' | wc -l]; my $MSG_ACTIVE = 'There are active NDMP sessions'; my $MSG_STOPNDMP = 'Prior to any upgrade activity, all NDMP sessions must complete or be stopped from the associated backup application.'; my $SPF_COMPATIBILITY = 'Make sure to verify compatibility of your NDMP backup application with the destination OneFS %s prior to the scheduled upgrade. For details on compatibility review the Isilon OneFS for Backup & Archive Simple Support Matrix [https://elabnavigator.emc.com/vault/pdf/PowerScale_Isilon_OneFS_Backup_Archive.pdf]'; # Gather my $gather = []; if ($RPS{'Live'}) { if (isCurrentOneFS('>=8')) { $gather = gatherLocal($CMD_SESSIONS_8); } else { $gather = gatherLocal($CMD_SESSIONS_7); } } # Analyze my $active_ndmp = 0; foreach my $line (@$gather) { if ($line =~ /^\s*\d.*ACTIVE/ or $line =~ /^\s*\d.* A /) { $active_ndmp = 1; last; } } # Gather $gather = gatherLocal(isLive()?$CMD_LIVE_TRACE:$CMD_LOGS_TRACE); # Analyze my $recent_trace = 0; foreach my $line (@$gather) { if ($line =~ /^(\d+)$/) { $recent_trace = $1; } } # Flag if ($active_ndmp) { critical($MSG_ACTIVE) } if (isServiceEnabled('ndmpd')) { addCustomerInformation(info($MSG_STOPNDMP)); if (isOneFSUpgrade()) { addCustomerInformation (info(sprintf($SPF_COMPATIBILITY, isDestinationOneFS()))); } } # Output Check result printResults(); } # End checkNDMP $CHECKS{'checkNetBIOS'}{'Description'} = 'Checks if the Isilon NetBIOS Name'. ' Service (nbns) is enabled when updating to OneFS 8.0.1 and later.'; sub checkNetBIOS { if (wasCalled()) { return } if (isCurrentOneFS('>=8.0.1')) { return unsupported('Supported in OneFS 8.0.0.x and earlier'); } if (!$RPS{'Target'} or compareVersions($RPS{'Target'}, '8') <= 0) { return unsupported('Supported when the parameter target_version is set'. ' to OneFS 8.0.1 or later'); } # Setup printCheck('NetBIOS EOL'); my $MSG_EOSL = 'The Isilon NetBIOS Name Service (nbns) is enabled and is'. ' not supported beyond OneFS 8.0.0.x. Review'. ' https://support.emc.com/docu69808 - End of Life (EOL) Announcement'. ' for the EMC Isilon NetBIOS Name Service.pdf for details.'; # Flag if (isServiceEnabled('nbns')) { addCustomerInformation(fail($MSG_EOSL)); } # Output printResults(); } # checkNFS: Runs nfsstat and looks for Encountered RPC error $CHECKS{'checkNFS'}{'Description'} = 'Uses nfsstat to identify RPC errors'; sub checkNFS { if (wasCalled()) { return } printCheck('NFS'); if (isServiceDisabled('nfs')) { return printResults('Disabled') } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'nfsstat']. q[| grep 'Encountered RPC error']; my $CMD_LOGS = q[2>&1 grep -H 'Encountered RPC error' */nfsstat]; my $SPF_ISSUES = q[NFS issues detected on the following nodes: %s]; checkNFS8004(); # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my %rpc_errors = (); foreach my $lnn (@LNNS) { foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /Encountered RPC error/) { $rpc_errors{$lnn} = 1; } } } # Flag if (%rpc_errors) { critical(sprintf($SPF_ISSUES, compressRange(keys %rpc_errors))); } # Output printResults(); } # End checkNFS $CHECKS{'checkOpenFiles'}{'Description'} = 'Checks each node for the number of'. ' open files [sysctl kern.openfiles] and compares that against the number'. ' of max open files [sysctl kern.maxfiles]. The check will warn at over'. ' 80% of the max and fail at over 90% of the max.'; sub checkOpenFiles { if (wasCalled()) { return } printCheck('Kernel Open Files Count'); # Setup my $MAX = 315000; my $SPF = '%5s %15s %15s'; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'sysctl kern.openfiles kern.maxfiles']; my $CMD_LOGS = q[2>&1 zgrep -HE 'kern.openfiles|kern.maxvnodes']. q[ */kern.openfiles */varlog.tar/log/vmlog */varlog.tar/log/vmlog.0.gz]; my $SPF_APP = qq[The following nodes are approaching their max number of]. q[ open files: %s]; my $SPF_OVER = qq[The following nodes have exceeded their max number of]. q[ open files: %s]; my $MSG_DOC = q[For details review: https://dl.dell.com/content/docu94152]. q[ - PowerScale OneFS Technical Specifications Guide]; # Gather my $gather = gatherCluster(($RPS{'Live'}?$CMD_LIVE:$CMD_LOGS)); # Analyze my @over_80 = (); my @over_90 = (); my @over_100 = (); verboseHeader(sprintf($SPF, 'LNN', 'Open Files', 'Max Open Files')); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } my $maxfiles = $MAX; my $openfiles = 0; foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /kern.openfiles:\s+(\d+)$/) { $openfiles = $1 } if ($line =~ /kern.maxfiles:\s+(\d+)$/) { $maxfiles = $1 } if ($line =~ /kern.maxvnodes:\s+(\d+)$/) { $maxfiles = $1 * 0.9 } } verbose(sprintf($SPF, $lnn, $openfiles, $maxfiles)); if ($openfiles > $maxfiles) { push(@over_100, $lnn) } elsif ($openfiles > $maxfiles * 0.9) { push(@over_90, $lnn) } elsif ($openfiles > $maxfiles * 0.8) { push(@over_80, $lnn) } } # Flag if (@over_80) { warning(sprintf($SPF_APP, compressRange(@over_80))) } if (@over_90) { fail(sprintf($SPF_APP, compressRange(@over_90))) } if (@over_100) { fail(sprintf($SPF_OVER, compressRange(@over_100))) } if (isResults(&WARN)) { info($MSG_DOC) } # Output printResults(); } # End checkOpenFiles $CHECKS{'checkPartitions'}{'Description'} = "Checks system partition space."; sub checkPartitions { return if wasCalled(); printCheck("System Partition Free Space"); my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster("2>&1 ${SUDO} isi_for_array df -Pli"); } else { $gather = gatherCluster("2>&1 grep -H . */df"); } foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } my $mirror_node = 0; my $imdd_node = 0; my $num_blocks = 0.00048828125; foreach my $line (@{$gather->{$lnn}}) { verbose($line); $mirror_node = 1 if $line =~ /^\s*\/dev\/mirror\//; $imdd_node = 1 if $line =~ /^\s*\/dev\/imdd/; if ($line =~ /Filesystem\s+(\S+)/) { if ($1 eq "512-blocks") { $num_blocks = 0.00048828125; } elsif ($1 eq "1K-blocks") { $num_blocks = 0.0009765625; } } next unless ($line =~ /\s*(\/.*?)\s+(\d+)\s+(\d+)\s+(-?\d+)\s+(\d+)%\s+(\d+)\s+(\d+)\s+(\d+)%\s+(\S+.*)$/); my $fs = $1; my $total = int($2 * $num_blocks); my $used = int($3 * $num_blocks); my $available = int($4 * $num_blocks); my $capacity = $5; my $iused = $8; my $mount = $9; debug("$fs,$total,$used,$capacity,$iused,$mount"); # root (/) partition if ($mount eq "/") { # Flag over 97% full if ($capacity > 97) { fail("Node $lnn partition $mount is over 97% full."); } # Check less than 250M free if ($available < 250) { push(@{$RPS{'root<250'}}, $lnn); } } # Flag /var partition over 90% full if ($mount eq "/var" and $capacity > 90) { fail("Node $lnn partition $mount is over 90% full."); } # Flag /var/crash partition over 50/90% full if ($mount eq "/var/crash" and $capacity > 90) { fail("Node $lnn partition $mount is over 90% full."); } elsif ($mount eq "/var/crash" and $capacity > 50) { warning("Node $lnn partition $mount is over 50% full."); } # Flag inode usage over 90% if (($mount eq "/" or $mount eq "/var" or $mount eq "/var/crash") and $iused > 97) { fail("Node $lnn partition $mount inode usage is over 97%."); } # Flag var less than 500MB free space for use in node firmware check if ($mount eq '/var' and $available < 500) { push(@{$RPS{'Node Firmware'}{'var10'}}, $lnn); } # Flag var less than 100MB free space for use in node firmware check if ($mount eq '/var' and $available < 100) { push(@{$RPS{'Node Firmware'}{'var11'}}, $lnn); } # Node compatibility check for 8.0.0.7 and IQ nodes if ($mount eq "/" and $total < 500) { $RPS{'Root less than 500MB'}{$lnn} = 1; } # Node compatibility check for A100 nodes if ($mount eq "/" and $total < 1000) { $RPS{'Root less than 1GB'}{$lnn} = 1; } } if ($mirror_node) { $NODE_INFO{$lnn}{'Bootflash'} = 1; debug("NODE_INFO->${lnn}->Bootflash=1"); push(@{$CLUSTER_INFO{'Bootflash'}}, $lnn); debug("CLUSTER_INFO->Bootflash=$lnn"); } if ($imdd_node) { $NODE_INFO{$lnn}{'IMDD'} = 1; debug("NODE_INFO->${lnn}->IMDD=1"); push(@{$CLUSTER_INFO{'IMDD'}}, $lnn); debug("CLUSTER_INFO->IMDD=$lnn"); } } info("Refer to KB 335086 (https://www.dell.com/support/kbdoc/335086) for details.") if isResults(&WARN); printResults(); } # End checkPartitions $CHECKS{'checkPerformance'}{'Description'} = "Checks cluster performance"; sub checkPerformance { return if wasCalled(); checkHardwareStatus(); printCheck("Performance"); # Pre-defined threshold for performance checks my $THRESHOLD = 80; # A list of the last 7 days my @abbr = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec); my @date = (); for (my $i = 0; $i <= 7; $i++) { my ($day, $month) = (localtime($RPS{'Timestamp'} - ($i * 86400)))[3,4]; push(@date, sprintf("%3s %2s", $abbr[$month], $day)); } my $dategrep = join("|", @date); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array "zgrep -H -E '^Mon|^Tue|^Wed|^Thu|^Fri|^Sat|^Sun|vm.loadavg' /var/log/vmlo* | paste - - | grep -E '$dategrep'"]); } else { $gather = gatherCluster(qq[2>&1 zgrep -H -E '^Mon|^Tue|^Wed|^Thu|^Fri|^Sat|^Sun|vm.loadavg' */varlog.tar/log/vmlo* | paste - - | grep -E '$dategrep']); } # Analyze information for issues. my %stats = (); my %average_above_threshold = (); my ($total_of_averages, $total_of_highest, $node_count); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } # Parse command output foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ /vm.loadavg: \{ \S+ \S+ ([0-9.]+) \}/; $stats{$lnn}{'load'} += $1; $stats{$lnn}{'count'}++; if (!$stats{$lnn}{'highest'} or $1 > $stats{$lnn}{'highest'}) { $stats{$lnn}{'highest'} = $1; } } next unless exists $stats{$lnn}{'count'} and exists $stats{$lnn}{'highest'} and exists $stats{$lnn}{'load'}; # Calculate the average load $stats{$lnn}{'average'} = $stats{$lnn}{'load'} / $stats{$lnn}{'count'}; next unless $NODE_INFO{$lnn}{'Cores'}; # Assign the number of cores for this node my $core_count = $NODE_INFO{$lnn}{'Cores'}; # Turn the performance loads into percents based on the number of cores for the node my $highest = sprintf("%d", ($stats{$lnn}{'highest'} / $core_count) * 100); my $average = sprintf("%d", ($stats{$lnn}{'average'} / $core_count) * 100); # Set a flag if the average is above threshold if ($average > $THRESHOLD) { $average_above_threshold{$lnn} = 1; } # Increment cluster totals $total_of_averages += $average; $total_of_highest += $highest; $node_count++; # Verbose output verbose("Highest: ${highest}%"); verbose("Average: ${average}%"); } # Warn if any flag is set for an average being above threshold if (%average_above_threshold) { warning("The following nodes have an average load of over ${THRESHOLD}%: ". compressRange(keys %average_above_threshold)); } # Assign average performance across cluster my ($average_of_averages, $average_of_highest); $average_of_averages = int($total_of_averages / $node_count) if defined $total_of_averages and defined $node_count; $average_of_highest = int($total_of_highest / $node_count) if defined $total_of_highest and defined $node_count; # Verbose output verboseHeader($CLUSTER_INFO{'Name'}); verbose("Average of highest across the cluster: ${average_of_highest}%") if defined $average_of_highest; verbose("Average of averages across the cluster: ${average_of_averages}%") if defined $average_of_averages; # Output Check result printResults(); } # End checkPerformance ### # checkProcesses: # Checks whether a node owns the OpenSM master role. One node in the cluster must own the OpenSM master role. # https://www.dell.com/support/kbdoc/16664 # # Checks for MCP Running on all nodes # https://www.dell.com/support/kbdoc/175249 - OneFS: isi_mcp fails to start on one or more nodes in the cluster $CHECKS{'checkProcesses'}{'Description'} = "Checks for OpenSM master, MCP, and isi_upgrade_d process related issues"; sub checkProcesses { return if wasCalled(); printCheck("Processes"); gatherNetworkInterfaces(); # Gather information my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'ps auxww | grep -E "isi_mcp|opensm|isi_upgrade_d" | grep -v grep']); } else { $gather = gatherCluster(qq[2>&1 grep -H -E "isi_mcp|opensm|isi_upgrade_d" */ps]); } # Analyze information for issues. my $opensm_master = 0; my @nodes_with_upgrade_process = (); my @nodes_with_not_enough_isi_mcp = (); foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } my $mcp_processes = 0; foreach my $line (@{$gather->{$lnn}}) { verbose($line); # Check for two isi_mcp processes on each node: $mcp_processes++ if ($line =~ /isi_mcp/); # Check for an OpenSM Master if ($line =~ /opensm:\s+(0x\S+)\s+master/) { $opensm_master++; } # Check for isi_upgrade_d processes, there should be none running push(@nodes_with_upgrade_process, $lnn) if ($line =~ /isi_upgrade_d/); } # Check that we found a master and child isi_mcp process push(@nodes_with_not_enough_isi_mcp, $lnn) if ($mcp_processes < 2); } warning("isi_upgrade_d process running on the following nodes:".join(",", @nodes_with_upgrade_process)) if (@nodes_with_upgrade_process); fail("isi_mcp process count less than 2 on the following nodes:".join(",", @nodes_with_not_enough_isi_mcp)) if (@nodes_with_not_enough_isi_mcp); # Check that opensm master processes matches the number of IB Interfaces on each node if ($CLUSTER_INFO{'Active IB Interfaces'} and $opensm_master != $CLUSTER_INFO{'Active IB Interfaces'}) { fail("Number of OpenSM Masters does not match the number of InfiniBand Interfaces in use"); } info("Refer to KB 335094 (https://www.dell.com/support/kbdoc/335094) for details.") if isResults(&WARN); # Output Check result printResults(); } # End checkProcesses sub checkRFC2307 { return if wasCalled(); verboseHeader("RFC2307 Settings"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi_gconfig|grep ActiveDirectory"); } elsif (-e "local/isi_gconfig") { $gather = gatherLocal("2>&1 grep ActiveDirectory local/isi_gconfig"); } elsif (-e "local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc") { $gather = gatherLocal("2>&1 grep ActiveDirectory local/gconfig_ifsvar.tar/ifs/_ifsvar/main_config.gc"); } # Analyze information for issues. my ($sfu_support_global, $allocate_gids_global, $allocate_uids_global, $store_sfu_mappings_global); my %ADS = (); foreach (@$gather) { if (/ActiveDirectory\.Instance\._key\.([^\.]+)\./) { my $id = $1; if (/$id\._name .*= (\S+)$/) { $ADS{$id}{'Name'} = $1; } elsif (/$id\._dword\.AllocateGids\.value .*= (\S+)$/) { $ADS{$id}{'AllocateGids'} = $1; } elsif (/$id\._dword\.AllocateUids\.value .*= (\S+)$/) { $ADS{$id}{'AllocateUids'} = $1; } elsif (/$id\._sz\.SFUSupport\.value .*= (\S+)$/) { $ADS{$id}{'SFUSupport'} = $1; } elsif (/$id\._dword\.StoreSFUMappings\.value .*= (\S+)$/) { $ADS{$id}{'StoreSFUMappings'} = $1; } } elsif (/ActiveDirectory\.AllocateGids .*= (\S+)$/) { $allocate_gids_global = $1; } elsif (/ActiveDirectory\.AllocateUids .*= (\S+)$/) { $allocate_uids_global = $1; } elsif (/ActiveDirectory\.SFUSupport .*= (\S+)$/) { $sfu_support_global = $1; } elsif (/ActiveDirectory\.StoreSFUMappings .*= (\S+)$/) { $store_sfu_mappings_global = $1; } } # Setup global/defaults if they were not gathered $allocate_gids_global = "1" unless defined $allocate_gids_global; $allocate_uids_global = "1" unless defined $allocate_uids_global; $sfu_support_global = "none" unless defined $sfu_support_global; $store_sfu_mappings_global = "0" unless defined $store_sfu_mappings_global; # Update ADS values if they were not gathered to the global/defaults and then see if it meets criteria to be impacted by mapping issues my @ads_with_issues = (); verbose(sprintf("%-20s %15s %15s %15s %16s", "Provider Name", "AllocateGids", "AllocateUids", "SFUSupport", "StoreSFUMappings")); verbose(sprintf("%-20s %15s %15s %15s %16s", "Global Setttings", $allocate_gids_global, $allocate_uids_global, $sfu_support_global, $store_sfu_mappings_global)); foreach my $id (keys %ADS) { $ADS{$id}{'Name'} = $id unless defined $ADS{$id}{'Name'}; $ADS{$id}{'AllocateGids'} = $allocate_gids_global unless defined $ADS{$id}{'AllocateGids'}; $ADS{$id}{'AllocateUids'} = $allocate_uids_global unless defined $ADS{$id}{'AllocateUids'}; $ADS{$id}{'SFUSupport'} = $sfu_support_global unless defined $ADS{$id}{'SFUSupport'}; $ADS{$id}{'StoreSFUMappings'} = $store_sfu_mappings_global unless defined $ADS{$id}{'StoreSFUMappings'}; verbose(sprintf("%-20s %15s %15s %15s %16s", $ADS{$id}{'Name'}, $ADS{$id}{'AllocateGids'}, $ADS{$id}{'AllocateUids'}, $ADS{$id}{'SFUSupport'}, $ADS{$id}{'StoreSFUMappings'})); next unless $ADS{$id}{'SFUSupport'} =~ /rfc2307/i; next unless $ADS{$id}{'AllocateGids'} =~ /1/ or $ADS{$id}{'AllocateUids'} =~ /1/; push(@ads_with_issues, $ADS{$id}{'Name'}); } warning("The following ADS Authentication Providers are showing as configured for RFC2307 and are also enabled to auto-assign UIDs/GIDs: ".join(", ", @ads_with_issues)) if @ads_with_issues; addCustomerInformation("It was detected that an ADS Authentication Provider is configured for RFC2307 support and Auto-assign UIDs/GIDs are also enabled. This may lead to mapping issues which may impact client authentication to the cluster. See KB335338 (https://www.dell.com/support/kbdoc/335338) for more information on how to configure OneFS and Active Directory for RFC2307 compliance.") if @ads_with_issues; } # End checkRFC2307 $CHECKS{'checkServicesMonitoring'}{'Description'} = 'Checks that enabled services are being monitored'; sub checkServicesMonitoring { if (wasCalled()) { return } printCheck('Services - Monitoring'); gatherServices(); # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'grep -H ignore /etc/mcp/override/*']; my $CMD_LOGS = q[for i in $(grep -l ignore */etc.tar/etc/mcp/override/*);]. q[ do echo $i: $(grep -H ignore $i); done 2>&1]; my $SPF_IGNORE = q[The service %s is enabled but set to ignore on the]. q[ following nodes: %s]; my $MSG_INFO = q[The Master Control Process (MCP|isi_mcp) will not]. q[ monitor services flagged for ignore. Those services must be]. q[ manually started/stopped on each node. This also implies that if a]. q[ node is rebooted, MCP will not automatically start the service]. q[ on that node and it must be manually started. To allow MCP to]. q[ resume monitoring a service run the following command replacing]. q[ with the appropriate service name: isi services -a]. q[ monitor]; # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my %ignored_services = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (!/attribute="ignore"/ or !/value="1"/) { next } my $service = ''; if (/\/etc\/mcp\/override\/(\S+):/) { $service = $1 } debug("Ignored service: $service"); if (isServiceEnabled($service)) { push(@{$ignored_services{$service}}, $lnn); } } } # Flag foreach my $service (keys %ignored_services) { fail(sprintf($SPF_IGNORE, $service, compressRange(@{$ignored_services{$service}}))); } # Output if (isResults(&FAIL)) { info($MSG_INFO) } printResults(); } # End checkServicesMonitoring $CHECKS{'checkServices'}{'Description'} = 'Checks common services to make sure they are in the expected states'; sub checkServices { if (wasCalled()) { return } printCheck('Cluster Services'); gatherServices(); if (!$CLUSTER_INFO{'Services'}) { fail('Unable to gather cluster services information'); printResults(); return; } gatherSyncIQ(); # Output details about the status of the cluster services verboseHeader('Cluster Services'); verbose(sprintf('%-30s %s', 'Name', 'Status')); verbose(sprintf('%-30s %s', '-'x30, '-'x10)); verbose(sprintf('%-30s %s', $_, $CLUSTER_INFO{'Services'}{$_})) foreach (sort keys %{$CLUSTER_INFO{'Services'}}); # Check if critical services are disabled fail('Authentication (auth) service is disabled.') if isServiceDisabled('auth'); fail('Cluster Event Log (celog) service is disabled.') if isServiceDisabled('celog'); fail('Cluster Event Log Coalescer (celog_coalescer) service is disabled.') if isServiceDisabled('celog_coalescer'); fail('Cluster Event Log System Monitor (celog_monitor) service is disabled.') if isServiceDisabled('celog_monitor'); fail('Cluster Event Log Notification Daemon (celog_notification) service is disabled.') if isServiceDisabled('celog_notification'); fail('Filesystem Daemon (ifsd) service is disabled.') if isServiceDisabled('ifsd'); fail('Drive Daemon (isi_drive_d) service is disabled.') if isServiceDisabled('isi_drive_d'); fail('Drive Repurposing Daemon (isi_drive_repurpose_d) service is disabled.') if isServiceDisabled('isi_drive_repurpose_d'); fail('Platform API Daemon (isi_papi_d) service is disabled.') if isServiceDisabled('isi_papi_d'); fail('Group Change Daemon (isi_group_change_d) service is disabled.') if isServiceDisabled('isi_group_change_d'); fail('Job Daemon (isi_job_d) service is disabled.') if isServiceDisabled('isi_job_d'); fail('MCP Failsafe Daemon (isi_mcp) service is disabled.') if isServiceDisabled('isi_mcp'); fail('Protocol Availability (isi_protohealth_d) service is disabled.') if isServiceDisabled('isi_protohealth_d'); fail('RPC Daemon (isi_rpc_d) service is disabled.') if isServiceDisabled('isi_rpc_d'); fail('Likewise Service Manager (lwsm) service is disabled.') if isServiceDisabled('lwsm'); warning('Tardis Configuration (isi_tardis_d) service is disabled.') if isServiceDisabled('isi_tardis_d'); warning('Service isi_upgrade_d is currently enabled.', ' The isi_upgrade_d service should only be enabled when performing'. ' a OneFS upgrade and coming from a OneFS version prior to OneFS 8.') if isServiceEnabled('isi_upgrade_d'); if (isServiceDisabled('isi_upgrade_agent_d')) { my $msg = 'Service isi_upgrade_agent_d is currently disabled. The'. ' isi_upgrade_agent_d service must be enabled prior to performing'. ' any upgrade activities in OneFS 8 and later codes.'; if (isDestinationOneFS() or $RPS{'UpgradePlan'}) { critical($msg); } else { warning($msg); } } if (isServiceDisabled('isi_migrate') and ($CLUSTER_INFO{'SyncIQ'}{'Source'} or $CLUSTER_INFO{'SyncIQ'}{'Target'})) { critical('The SyncIQ Service (isi_migrate) is disabled. This service'. ' is required for SyncIQ functionality. To enable the'. ' service run the command: isi services isi_migrate enable'); } # Issue where upgrade commit will error if isi_cpool_d or isi_cpool_io_d services are disabled if (isCurrentOneFS('<8.2') and $RPS{'Target'} and compareVersions($RPS{'Target'}, '8.2') >= 0) { my @services = (); if (isServiceDisabled('isi_cpool_d')) { push(@services, 'isi_cpool_d') } if (isServiceDisabled('isi_cpool_io_d')) { push(@services, 'isi_cpool_io_d') } if (@services) { critical('For upgrades going to OneFS 8.2 and later, CloudPools services should be enabled to avoid encountering an error during the upgrade commit. '. "The following CloudPools services were found as disabled: @services"); } } # Issue where post HealthCheck package install the isi_healthcheck service is disabled critical('Healthcheck Daemon (isi_healthcheck) is not enabled and will need'. ' to be enabled to run any health checks via isi healthcheck commands.') if hasPatch('HealthCheck') and isServiceDisabled('isi_healthcheck'); # Issue where post upgrade to OneFS 8, isi_cbind_d service goes missing, KB 488106 fail('isi_cbind_d service is missing in isi services -a, manually verify and,'. ' if missing, escalate and reference KB 488106') if isCurrentOneFS(qr/^8/) and !$CLUSTER_INFO{'Services'}{'isi_cbind_d'}; # Display the check results printResults(); } # End checkServices sub checkSnapshotLog { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array 'awk "]. q[/EINVAL/ {EINVAL++; EINVALMR=\$1}]. q[/EIO/ && !/ifs_snap_delete/{EIO++; EIOMR=\$1}]. q[/EDEADLK/ {EDEADLK++; EDEADLKMR=\$1}]. q[/Failed to create snapshot/ {FAIL++; FAILMR=\$1}]. q[END {]. q[print \"EINVAL:\" EINVAL;]. q[print \"EINVALMR:\" EINVALMR;]. q[print \"EIO:\" EIO;]. q[print \"EIOMR:\" EIOMR;]. q[print \"EDEADLK:\" EDEADLK;]. q[print \"EDEADLKMR:\" EDEADLKMR;]. q[print \"FAIL:\" FAIL;]. q[print \"FAILMR:\" FAILMR;]. q[}]. q[" /var/log/isi_snapshot_d.log']); } else { $gather = gatherCluster(qq[for i in */varlog.tar/log/isi_snapshot_d.log;do ]. qq[echo -n \$i; awk '/EINVAL/ {EINVAL++; EINVALMR=\$1} END {print ":EINVAL:" EINVAL " EINVALMR:" EINVALMR}' \$i;]. qq[echo -n \$i; awk '/EIO/ && !/ifs_snap_delete/ {EIO++; EIOMR=\$1} END {print ":EIO:" EIO " EIOMR:" EIOMR}' \$i;]. qq[echo -n \$i; awk '/EDEADLK/ {EDEADLK++; EDEADLKMR=\$1} END {print ":EDEADLK:" EDEADLK " EDEADLKMR:" EDEADLKMR}' \$i;]. qq[echo -n \$i; awk '/Failed to create snapshot/ {FAIL++; FAILMR=\$1} END {print ":FAIL:" FAIL " FAILMR:" FAILMR}' \$i;]. qq[done]); } # Analyze information for issues. my %errors = (); foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $gather->{$lnn}; foreach my $line (@{$gather->{$lnn}}) { $errors{$lnn}{'EINVAL'}{'Count'} = $1 if ($line =~ /EINVAL:(\d+)/); $errors{$lnn}{'EINVAL'}{'MR'} = $1 if ($line =~ /EINVALMR:(\S+)/); $errors{$lnn}{'EIO'}{'Count'} = $1 if ($line =~ /EIO:(\d+)/); $errors{$lnn}{'EIO'}{'MR'} = $1 if ($line =~ /EIOMR:(\S+)/); $errors{$lnn}{'EDEADLK'}{'Count'} = $1 if ($line =~ /EDEADLK:(\d+)/); $errors{$lnn}{'EDEADLK'}{'MR'} = $1 if ($line =~ /EDEADLKMR:(\S+)/); $errors{$lnn}{'FAIL'}{'Count'} = $1 if ($line =~ /FAIL:(\d+)/); $errors{$lnn}{'FAIL'}{'MR'} = $1 if ($line =~ /FAILMR:(\S+)/); } } foreach my $lnn (keys %errors) { foreach my $type (keys %{$errors{$lnn}}) { warning("Node $lnn: Found $errors{$lnn}{$type}{'Count'} $type errors, the most recent was $errors{$lnn}{$type}{'MR'}"); } } } # End checkSnapshotLog ### # checkSnapshot: Checks some basic snapshot details $CHECKS{'checkSnapshot'}{'Description'} = "Checks if the Snapshot count is approaching cluster limit of 20,000, whether Autodelete is set to yes, and checks snapshot logs. Checks snapshot logs for EIN/EIO/EDEADLK/Failed to create snapshot"; sub checkSnapshot { return if wasCalled(); printCheck("Snapshot"); gatherSnapshotCount(); gatherSnapshotSettings(); gatherSnapshotUsage(); checkSnapshotLog(); if ($CLUSTER_INFO{'Snapshots'}{'Count'} and $CLUSTER_INFO{'Snapshots'}{'Count'} >= 20000) { critical("Total snapshot count is $CLUSTER_INFO{'Snapshots'}{'Count'}, more than the limit for snapshots per cluster of 20,000."); } elsif ($CLUSTER_INFO{'Snapshots'}{'Count'} and $CLUSTER_INFO{'Snapshots'}{'Count'} >= 19000) { critical("Total snapshot count is $CLUSTER_INFO{'Snapshots'}{'Count'}, approaching the limit for snapshots per cluster of 20,000."); } elsif ($CLUSTER_INFO{'Snapshots'}{'Count'} and $CLUSTER_INFO{'Snapshots'}{'Count'} >= 15000) { warning("Total snapshot count is $CLUSTER_INFO{'Snapshots'}{'Count'}, approaching the limit for snapshots per cluster of 20,000."); } if ($CLUSTER_INFO{'Snapshots'}{'Autodelete'} and $CLUSTER_INFO{'Snapshots'}{'Autodelete'} !~ /Yes/) { warning("Snapshot Autodelete not set to Yes"); } # Output Check result printResults(); } # End checkSnapshot $CHECKS{'checkSPNs'}{'Description'} = "Displays a list of SPNs, only useful when run with --extra"; sub checkSPNs { return if wasCalled(); printCheck("SPN List"); if (isCurrentOneFS('<7.1')) { verbose("This check has not been validated for OneFS versions prior to OneFS 7.1."); printResults(); return; } # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { if (isCurrentOneFS('<8')) { $gather = gatherLocal(qq[for i in `${SUDO} isi auth ads list -a -z|awk '{print \$1}'`; do ${SUDO} isi auth ads spn list --domain=\$i; done 2>&1]); } else { $gather = gatherLocal(qq[for i in `${SUDO} isi auth ads list -a -z|awk '{print \$1}'`; do ${SUDO} isi auth ads spn list \$i; done 2>&1]); } } else { $gather = gatherLocal(qq[2>&1 awk '/auth ads spn list/, /ads trusts list/' local/isi_auth]); } # Analyze information for issues. verboseHeader("SPN List"); foreach my $line (@$gather) { next if $line =~ /auth ads trusts list/; verbose($line); } # Output Check result printResults(); } # End checkSPNs sub checkSRSPools { if (wasCalled()) { return } gatherFlexNet(); gatherRemoteSupport(); gatherSRS(); my $MSG_SRS_NO_POOLS = 'SRS (isi esrs) is enabled, but no Gateway Access Pools are configured'; my $MSG_RS_NO_POOLS = 'Remote Support (isi remotesupport) is enabled, but no Gateway Access Pools are configured'; my $MSG_RS_NO_SUBNET = 'Remote Support (isi remotesupport) is enabled, but no Remote Support Subnet is configured'; my $SPF_SRS_MISSING_POOLS = 'SRS (isi esrs) has configured the following Gateway Access Pool(s) which could not be found: %s'; my $SPF_SRS_NON_SYSTEM = 'SRS (isi esrs) has configured the following Gateway Access Pool(s) which are not configured with a System Access Zone: %s'; # my $SPF_SRS_NON_STATIC = 'SRS (isi esrs) has configured the following Gateway Access Pool(s) which are not configured with a Static Allocation Method: %s'; my $SPF_RS_MISSING_POOLS = 'Remote Support (isi remotesupport) has configured the following Gateway Access Pool(s) which could not be found: %s. Instructions on how to resolve this may be found in KB 544862 [https://www.dell.com/support/kbdoc/544862 - Remote Support (isi remotesupport) has configured the following Gateway Access Pool(s) which could not be found]'; my $SPF_RS_MISSING_POOLS_UPG = 'If left unresolved the upgrade to OneFS %s may encounter an error shortly after the first node reboots.'; my $SPF_RS_NON_SYSTEM = 'Remote Support (isi remotesupport) has configured the following Gateway Access Pool(s) which are not configured with the System Access Zone: %s'; # my $SPF_RS_NON_STATIC = 'Remote Support (isi remotesupport) has configured the following Gateway Access Pool(s) which are not configured with a Static Allocation Method: %s'; my $SPF_RS_MISSING_SUBNET = 'Remote Support (isi remotesupport) has configured the following Remote Support Subnet which could not be found: %s'; # Flag if enabled but no pool or subnet configured if ($CLUSTER_INFO{'SRS'}{'Enabled'} and !$CLUSTER_INFO{'SRS'}{'Pools'}) { critical($MSG_SRS_NO_POOLS); } if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and !$CLUSTER_INFO{'RemoteSupport'}{'Pools'} and isCurrentOneFS('>=8') and isCurrentOneFS('<8.2')) { critical($MSG_RS_NO_POOLS); } if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and !$CLUSTER_INFO{'RemoteSupport'}{'Subnet'} and isCurrentOneFS('<8')) { critical($MSG_RS_NO_SUBNET); } # Checks that the network pools/subnets configured for remote connectivity # exist and are in the System access zone and Static allocation method if ($CLUSTER_INFO{'SRS'}{'Pools'} and $CLUSTER_INFO{'FlexNet'}{'Pools'}) { my @missing_pools = (); my @non_system_pools = (); my @non_static_pools = (); foreach my $pool (@{$CLUSTER_INFO{'SRS'}{'Pools'}}) { my @pools = grep(/\.$pool$/, keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}); if (!@pools) { push(@missing_pools, $pool); next; } my $found_system_pool = 0; foreach my $found_pool (@pools) { if ($CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Auth Zone'} and $CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Auth Zone'} eq 1) { $found_system_pool = 1; last; } } if (!$found_system_pool) { push(@non_system_pools, $pool) } # if ($CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Allocation Method'} and # $CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Allocation Method'} !~ qr/^static$/i) { # push(@non_static_pools, $pool); # } } if ($CLUSTER_INFO{'SRS'}{'Enabled'} and @missing_pools) { warning(sprintf($SPF_SRS_MISSING_POOLS, join(', ', @missing_pools))); } if ($CLUSTER_INFO{'SRS'}{'Enabled'} and @non_system_pools) { warning(sprintf($SPF_SRS_NON_SYSTEM, join(', ', @non_system_pools))); } # if ($CLUSTER_INFO{'SRS'}{'Enabled'} and @non_static_pools) { # warning(sprintf($SPF_SRS_NON_STATIC, join(', ', @non_static_pools))); # } } if ($CLUSTER_INFO{'RemoteSupport'}{'Pools'} and $CLUSTER_INFO{'FlexNet'}{'Pools'} and isCurrentOneFS('<8.2')) { my @missing_pools = (); my @non_system_pools = (); my @non_static_pools = (); foreach my $pool (@{$CLUSTER_INFO{'RemoteSupport'}{'Pools'}}) { my ($found_pool) = grep(/\.$pool$/, keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}); if (!$found_pool) { push(@missing_pools, $pool); next; } if ($CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Auth Zone'} and $CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Auth Zone'} ne 1) { push(@non_system_pools, $pool); } # if ($CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Allocation Method'} and # $CLUSTER_INFO{'FlexNet'}{'Pools'}{$found_pool}{'Allocation Method'} !~ qr/static/i) { # push(@non_static_pools, $pool); # } } if (@missing_pools) { my $msg_pools = sprintf($SPF_RS_MISSING_POOLS, join(', ', @missing_pools)); if (isDestinationOneFS('>=8.2') and isDestinationOneFS('<9.1')) { fail($msg_pools.' '. sprintf($SPF_RS_MISSING_POOLS_UPG, isDestinationOneFS())); } else { warning($msg_pools); } } if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and @non_system_pools) { warning(sprintf($SPF_RS_NON_SYSTEM, join(', ', @non_system_pools))); } # if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and @non_static_pools) { # warning(sprintf($SPF_RS_NON_STATIC, join(', ', @non_static_pools))); # } } if ($CLUSTER_INFO{'RemoteSupport'}{'Subnet'} and $CLUSTER_INFO{'FlexNet'}{'Subnets'} and !$CLUSTER_INFO{'FlexNet'}{'Subnets'}{$CLUSTER_INFO{'RemoteSupport'}{'Subnet'}}) { warning(sprintf($SPF_RS_MISSING_SUBNET, $CLUSTER_INFO{'RemoteSupport'}{'Subnet'})); } } $CHECKS{'checkSRS'}{'Description'} = 'Checks the remote connectivity configuration for issues'; sub checkSRS { if (wasCalled()) { return } printCheck('DialHome & Remote Connectivity'); gatherRemoteSupport(); gatherSRS(); gatherProductInfo(); checkSRSPools(); checkEventChannel(); my $MSG_SRS_SVC = 'SRS (isi esrs) is enabled, but the SRS daemon service is disabled and should be enabled. To enable the SRS Daemon service run the following command: isi services -a isi_esrs_d enable'; my $MSG_RS_SVC = 'RemoteSupport (isi remotesupport) is enabled, but the ConnectEMC service is disabled and should be enabled. To enable the ConnectEMC service run the following command: isi services -a connectemc enable'; my $MSG_BOTH = 'Both SRS (isi esrs) and RemoteSupport (isi remotesupport) methods for remote connectivity are enabled. The RemoteSupport (isi remotesupport) method should be disabled by running the following command: isi remotesupport connectemc modify --enabled=false'; my $MSG_PS_NODES = 'DialHomes and RemoteSupport (isi remotesupport) will not work on PowerScale F200 and PowerScale F600 nodes in OneFS 9.0.0.0 and later. Consider migrating to SRS (isi esrs) method for remote connectivity.'; my $MSG_RS_TO_SRS = 'RemoteSupport (isi remotesupport) is enabled, and the cluster is running a version of OneFS 8.1 or later. In OneFS 8.1 and later, SRS (isi esrs) is the recommended method for remote support functionality. Review the "Remote support" section of the administration guides for your current OneFS version for details on how to enable this functionality.'; my $MSG_KB = 'For additional details see KB 535894 [https://www.dell.com/support/kbdoc/535894 - Isilon On-Cluster Analysis - DialHome & Remote Connectivity]'; my $MSG_91 = 'The SRSv2 method for remote connectivity via "isi remotesupport connectemc" commands is enabled, this method must be disabled before upgrading to OneFS 9.1 or later. It is recommended to configure the SRSv3 method for remote connectivity via "isi esrs" commands or in the WebUI. See https://support.emc.com/docu95370 for more information on how to configure SRSv3.'; # SRSv3 (isi esrs) is enabled if ($CLUSTER_INFO{'SRS'}{'Enabled'}) { details('SRS (isi esrs) is enabled'); # Flag if the isi_esrs_d service is disabled if (isServiceDisabled('isi_esrs_d')) { critical($MSG_SRS_SVC) } } # Remote Support (isi remotesupport) is enabled if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'}) { details('RemoteSupport (isi remotesupport) is enabled'); # Flag if the connectemc service is disabled if (isServiceDisabled('connectemc')) { critical($MSG_RS_SVC) } # Flag if there are any Gen6.5 nodes and RemoteSupport is being used if ($CLUSTER_INFO{'Gen6.5'}) { fail($MSG_PS_NODES) } # Flag if the cluster is running OneFS 8.1+ and not using SRS if (isCurrentOneFS('>=8.1') and !$CLUSTER_INFO{'SRS'}{'Enabled'}) { if (isDestinationOneFS('>=9.1')) { addCustomerInformation('Remote Connectivity', fail($MSG_91)); } else { addCustomerInformation('Remote Connectivity', warning($MSG_RS_TO_SRS)); } } } # Both are enabled if ($CLUSTER_INFO{'SRS'}{'Enabled'} and $CLUSTER_INFO{'RemoteSupport'}{'Enabled'}) { fail($MSG_BOTH); } # Neither is enabled if (!$CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and !$CLUSTER_INFO{'SRS'}{'Enabled'}) { if (isCurrentOneFS('<7.1')) { details('RemoteSupport/SRS is not supported prior to OneFS 7.1'); } elsif (isCurrentOneFS('<9.1')) { details('RemoteSupport/SRS is not enabled'); } else { details('SRS is not enabled'); } } # Verbose output for isi esrs if (isCurrentOneFS('>=8.1')) { verboseHeader('SRSv3/SRS (isi esrs)'); verbose('Enabled: '. ($CLUSTER_INFO{'SRS'}{'Enabled'} or 'No')); verbose('Primary SRS Gateway: '. ($CLUSTER_INFO{'SRS'}{'Primary Gateway'} or '-')); verbose('Secondary SRS Gateway: '. ($CLUSTER_INFO{'SRS'}{'Secondary Gateway'} or '-')); verbose('Gateway Access Pools: '. ($CLUSTER_INFO{'SRS'}{'Pools'}? "@{$CLUSTER_INFO{'SRS'}{'Pools'}}":'-')); } # Verbose output for isi remotesupport if (isCurrentOneFS('>=7.1') and isCurrentOneFS('<9.1')) { verboseHeader('SRSv2/RemoteSupport (isi remotesupport):'); verbose('Enabled: '. ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} or 'No')); verbose('Primary SRS Gateway: '. ($CLUSTER_INFO{'RemoteSupport'}{'Primary Gateway'} or '-')); verbose('Secondary SRS Gateway: '. ($CLUSTER_INFO{'RemoteSupport'}{'Secondary Gateway'} or '-')); if (isCurrentOneFS('>=8')) { verbose('Gateway Access Pools: '. ($CLUSTER_INFO{'RemoteSupport'}{'Pools'}? "@{$CLUSTER_INFO{'RemoteSupport'}{'Pools'}}":'-')); } else { verbose('Remote Support Subnet: '. ($CLUSTER_INFO{'RemoteSupport'}{'Subnet'} or '-')); } } # Output results if (isResults(&WARN)) { info($MSG_KB) } printResults(); } # End checkSRS sub checkSSHDConfigSHA256 { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2')) { return } if (!isDestinationOneFS('>=8.2')) { return } if (!isLive()) { return } if (isHardened()) { return } # Setup my $CMD_LIVE = "2>&1 ${SUDO} isi_for_array 'sha256 -q /etc/ssh/sshd_config'"; my $MSG_FAIL = 'A mismatch of /etc/ssh/sshd_config files was found. For details on'. ' how to resolve this review KB 000169490'. ' [https://www.dell.com/support/kbdoc/en-us/000169490 - When running'. ' health check, you may get critical error "SSHd config'. ' (/etc/ssh/sshd_config) on nodes have different MD5 hashes than'. ' others"'; # Gather my $gather = gatherCluster($CMD_LIVE); # Analyze my %sha256sums = (); my $spf = '%5s %s'; verboseHeader('/etc/ssh/sshd_config SHA256 Checksum Values'); verbose(sprintf($spf, 'LNN', 'SHA256 Checksum Value')); verbose(sprintf($spf, '-'x5, '-'x64)); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $checksum (@{$gather->{$lnn}}) { $checksum =~ s/^\s*|\s*$//; if ($checksum !~ /^[a-f0-9]{64}$/) { next } verbose(sprintf($spf, $lnn, $checksum)); $sha256sums{$checksum}++; } } # Flag if (scalar(keys %sha256sums) > 1) { fail($MSG_FAIL) } } # End checkSSHDconfigSHA256 $CHECKS{'checkSSHDConfig'}{'Description'} = "Checks the /etc/mcp/templates/sshd_config files for known issues"; sub checkSSHDConfig { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2')) { return } if (isCurrentOneFS('<7') and !$RPS{'Live'}) { return } printCheck("SSHD Config Compatibility"); checkSSHDConfigSHA256(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { my $cmd = q[cat /etc/mcp/templates/sshd_config]; $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array '$cmd']); } else { $gather = gatherCluster(qq[2>&1 grep -H . */etc.tar/etc/mcp/templates/sshd_config]); } # MACs no longer supported in OpenSSH 7.6 my @OpenSSH76RemovedMACs = ( 'hmac-ripemd160', 'hmac-ripemd160@openssh.com', 'hmac-ripemd160-etm@openssh.com', ); # Ciphers no longer supported in OpenSSH 7.6 my @OpenSSH76RemovedCiphers = ( 'blowfish-cbc', 'cast128-cbc', 'arcfour', 'arcfour128', 'arcfour256', ); # Analyze information for issues. my %bad_macs = (); my %bad_ciphers = (); my %appends = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { if ($line =~ /^\s*MACs\s*(\S+)$/) { my @MACs = split(/,/,$1); foreach my $MAC (@MACs) { my ($results) = grep {$_ eq $MAC} @OpenSSH76RemovedMACs; $bad_macs{$results} = 1 if $results; } } if ($line =~ /^\s*Ciphers\s*(\S+)$/) { my @Ciphers = split(/,/,$1); foreach my $Cipher (@Ciphers) { my ($results) = grep {$_ eq $Cipher} @OpenSSH76RemovedCiphers; $bad_ciphers{$results} = 1 if $results; } } if ($line =~ /^(\S+)\s+\+/) { $appends{$lnn}{$1}++; } } } # Flag issues detected if (%bad_ciphers or %bad_macs) { my $version = ""; if ($RPS{'Target'}) { $version = $RPS{'Target'}; } elsif (isCurrentOneFS('8.1.0.4') or isCurrentOneFS('8.1.2.0')) { $version = $CLUSTER_INFO{'OneFS'}; } if ($version eq "8.1.0.4" or $version eq "8.1.2.0") { fail("Support for the following Ciphers is removed in OpenSSH 7.7 and later which is included in the Kernel Rollup Patch for OneFS $version: ".join(", ", keys %bad_ciphers)) if %bad_ciphers; fail(" Support for the following MACs is removed in OpenSSH 7.7 and later which is included in the Kernel Rollup Patch for OneFS $version: ".join(", ", keys %bad_macs)) if %bad_macs; fail(" The compatibility issues found in the /etc/mcp/templates/sshd_config file should be resolved before applying the Kernel Rollup Patch."); fail(" If left unresolved the sshd service will fail to start."); } elsif ($RPS{'Target'} and isCurrentOneFS('<8.2') and compareVersions($RPS{'Target'}, "8.2") >= 0) { fail("Support for the following Ciphers is removed in OpenSSH 7.7 and later which is included in the upgrade to OneFS $RPS{'Target'}: ".join(", ", keys %bad_ciphers)) if %bad_ciphers; fail(" Support for the following MACs is removed in OpenSSH 7.7 and later which is included in the upgrade to OneFS $RPS{'Target'}: ".join(", ", keys %bad_macs)) if %bad_macs; fail(" The compatibility issues found in the /etc/mcp/templates/sshd_config file should be resolved before upgrading to $RPS{'Target'}."); fail(" If left unresolved the sshd service will fail to start."); } } my $multiple_append_issue = 0; foreach my $lnn (keys %appends) { foreach my $keyword (keys %{$appends{$lnn}}) { if ($appends{$lnn}{$keyword} > 1) { $multiple_append_issue = 1; } } } if ($multiple_append_issue and isCurrentOneFS('<8.2')) { if ($RPS{'Target'} and compareVersions($RPS{'Target'}, "8.2") >= 0) { fail("Multiple lines with appending values detected in /etc/mcp/templates/sshd_config"); } else { warning("Multiple lines with appending values detected in /etc/mcp/templates/sshd_config"); } } # Output Check result info("For additional details review KB537190 [https://www.dell.com/support/kbdoc/537190 - Isilon On-cluster Analysis - SSHD Config Compatibility]") if isResults(&WARN); printResults(); } # End checkSSHDConfig $CHECKS{'checkStoragePools'}{'Description'} = 'Check storage pools for'. ' health/capacity/unprovisioned drive issues'; sub checkStoragePools { if (wasCalled()) { return } # Setup printCheck('Storage Pools'); gatherIsiStatPools(); gatherNeighborhoods(); my $WARN_PERCENT = 90; my $FAIL_PERCENT = 95; my $SPF_UNPROV = 'There are unprovisioned drives: %s'; my $SPF_UNHEALTHY = 'The following pools are unhealthy: %s'; my $SPF_MISSING = 'The following pools are missing drives: %s'; my $SPF_HDD_FAIL = 'The following pools have HDD usage over 95%: %s'; my $SPF_HDD_WARN = 'The following pools have HDD usage over 90%: %s'; my $SPF_SSD_FAIL = 'The following pools have SSD usage over 95%: %s'; my $SPF_SSD_WARN = 'The following pools have SSD usage over 90%: %s'; my $SPF_USED_FAIL = 'The following pools have usage over 95%: %s'; my $SPF_USED_WARN = 'The following pools have usage over 90%: %s'; # Analyze my @fail_unhealthy_pools = (); my @missing_drives = (); my @fail_hdd_used_pools = (); my @warn_hdd_used_pools = (); my @fail_ssd_used_pools = (); my @warn_ssd_used_pools = (); my @fail_used_pools = (); my @warn_used_pools = (); foreach my $name (keys %{$CLUSTER_INFO{'Pools'}}) { my %pool = %{$CLUSTER_INFO{'Pools'}{$name}}; if ($pool{'Health'}) { if ($pool{'Health'} =~ /S|D|R|U/) { push(@fail_unhealthy_pools, $name); } if ($pool{'Health'} =~ /M/) { push(@missing_drives, $name); } } if ($pool{'Percent HDD Used'}) { if ($pool{'Percent HDD Used'} >= 95) { push(@fail_hdd_used_pools, $name); } elsif ($pool{'Percent HDD Used'} >= 90) { push(@warn_hdd_used_pools, $name); } } elsif ($pool{'Percent Used'}) { if ($pool{'Percent Used'} >= 95) { push(@fail_used_pools, $name); } elsif ($pool{'Percent Used'} >= 90) { push(@warn_used_pools, $name); } } if ($pool{'Percent SSD Used'}) { if ($pool{'Percent SSD Used'} >= 95) { push(@fail_ssd_used_pools, $name); } elsif ($pool{'Percent SSD Used'} >= 90) { push(@warn_ssd_used_pools, $name); } } } # Flag if ($CLUSTER_INFO{'Unprovisioned drives'}) { warning(sprintf($SPF_UNPROV, $CLUSTER_INFO{'Unprovisioned drives'})); } if (@fail_unhealthy_pools) { fail(sprintf($SPF_UNHEALTHY, join(', ', @fail_unhealthy_pools))); } if (@missing_drives) { warning(sprintf($SPF_MISSING, join(', ', @missing_drives))); } if (@fail_hdd_used_pools) { critical(sprintf($SPF_HDD_FAIL, join(', ', @fail_hdd_used_pools))); } if (@warn_hdd_used_pools) { warning(sprintf($SPF_HDD_WARN, join(', ', @warn_hdd_used_pools))); } if (@fail_ssd_used_pools) { critical(sprintf($SPF_SSD_FAIL, join(', ', @fail_ssd_used_pools))); } if (@warn_ssd_used_pools) { warning(sprintf($SPF_SSD_WARN, join(', ', @warn_ssd_used_pools))); } if (@fail_used_pools) { critical(sprintf($SPF_USED_FAIL, join(', ', @fail_used_pools))); } if (@warn_used_pools) { warning(sprintf($SPF_USED_WARN, join(', ', @warn_used_pools))); } # Output printResults(); } # End checkStoragePools ### # checkTimeDrift: Determines if the times on nodes are out of sync $CHECKS{'checkTimeDrift'}{'Description'} = "Checks time drift between nodes"; sub checkTimeDrift { return if wasCalled(); return unless $RPS{'Live'}; printCheck("Cluster Time Drift"); gatherUptime(); # Compare times between nodes, maintain a lowest and highest value my $min_time; my $max_time; foreach my $lnn (@LNNS) { next unless $NODE_INFO{$lnn}{'time'}; my $time = 0; $time += 12 * 60 if $NODE_INFO{$lnn}{'time'} =~ /PM/; $time += 12 * $1 + $2 if $NODE_INFO{$lnn}{'time'} =~ /^(\d+):(\d+)/; if (not defined $min_time) { $min_time = $max_time = $time; } if ($time < $min_time) { $min_time = $time; } if ($time > $max_time) { $max_time = $time; } } # Flag issues with time drift my $time_drift; my $results = "Unknown"; if (not defined $min_time or not defined $max_time) { fail("Unable to determine time drift"); } else { $time_drift = $max_time - $min_time; $results = "$time_drift Minutes"; if ($time_drift >= 5) { fail("Time drift between nodes is greater than 5 minutes"); } elsif ($time_drift >= 3) { warning("Time drift between nodes is between 3 and 5 minutes"); } } # Output Check result printResults($time_drift); } # End checkTimeDrift ### # checkTimeSync: Checks if the cluster is enabled to sync to an external server $CHECKS{'checkTimeSync'}{'Description'} = "Checks if the cluster is enabled to sync to an external server"; sub checkTimeSync { return if wasCalled(); printCheck("Cluster Time Sync"); gatherSMBTime(); gatherNTPConfig(); gatherNTPdXML(); # Warn if no external time sync is found if (!$CLUSTER_INFO{'NTP Servers'} and !$CLUSTER_INFO{'SMBTime'}) { warning("This cluster is not configured to synchronize time externally."); } # Output Check result printResults(); } # End checkTimeSync ### # checkTimeZone: Checks for missing time zone in target OneFS code level. $CHECKS{'checkTimeZone'}{'Description'} = "Checks for missing time zone in target OneFS code level."; sub checkTimeZone { if (wasCalled()) { return } printCheck("Cluster Time Zone"); gatherArrayXML(); # No target OneFS version, not impacted. if (!$RPS{'Target'}) { debug("checkTimeZone: No target OneFS version, not impacted."); } # Target OneFS version not a version known as impacted elsif ($RPS{'Target'} !~ /^8\.0\.0\.[012345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/) { debug("checkTimeZone: Target OneFS version not a version known as impacted"); } # Time zone was not gathered elsif (!$CLUSTER_INFO{'Timezone'}) { fail("Unable to determine time zone, manually verify"); } # Time zones missing in OneFS 8.0.0.1, OneFS 8.0.0.2, OneFS 8.0.0.3, OneFS 8.0.0.4, OneFS 8.0.0.5, OneFS 8.0.1.0, OneFS 8.0.1.1, and OneFS 8.1.0.0 elsif ($RPS{'Target'} =~ /^8\.0\.0\.[12345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/) { my @Timezone = ("Brazil/East","Brazil/DeNoronha","Brazil/West","Brazil/Acre","Pacific/Samoa","PRC","Mexico/General","Mexico/BajaSur","Mexico/BajaNorte","Singapore","Canada/Yukon","Canada/Pacific","Canada/Central","Canada/Eastern","Canada/Mountain","Canada/Atlantic","Canada/Newfoundland","Canada/East-Saskatchewan","Canada/Saskatchewan","America/Santa_Isabel","America/Ensenada","Chile/Continental","Chile/EasterIsland","GMT0","US/Arizona","US/Pacific","US/Central","US/Alaska","US/Eastern","US/Samoa","US/East-Indiana","US/Mountain","US/Aleutian","US/Indiana-Starke","US/Michigan","US/Hawaii","Antarctica/South_Pole","Asia/Rangoon"); if (grep {$_ eq $CLUSTER_INFO{'Timezone'}} @Timezone) { fail("$CLUSTER_INFO{'Timezone'} timezone found on this cluster, an upgrade to OneFS 8.0.0.[012345], 8.0.1.[01], or 8.1.0.0 should not be performed."); } } # Time zones missing in OneFS 8.0.0.0 elsif ($RPS{'Target'} =~ /^8\.0\.0\.0$/) { my @Timezone = ("Europe/Tiraspol","Europe/Belfast","Brazil/East","Brazil/DeNoronha","Brazil/West","Brazil/Acre","W-SU","GB-Eire","Pacific/Ponape","Pacific/Samoa","Pacific/Truk","Pacific/Yap","Israel","Portugal","ROC","Universal","PRC","Kwajalein","Iceland","ROK","Mexico/General","Mexico/BajaSur","Mexico/BajaNorte","Egypt","Navajo","Poland","Jamaica","Singapore","Africa/Timbuktu","Africa/Asmera","Australia/Canberra","Australia/South","Australia/NSW","Australia/North","Australia/Tasmania","Australia/West","Australia/Yancowinna","Australia/Victoria","Australia/ACT","Australia/LHI","Australia/Queensland","Libya","Canada/Yukon","Canada/Pacific","Canada/Central","Canada/Eastern","Canada/Mountain","Canada/Atlantic","Canada/Newfoundland","Canada/East-Saskatchewan","Canada/Saskatchewan","NZ","Japan","Zulu","Turkey","GMT+0","America/Jujuy","America/Buenos_Aires","America/Montreal","America/Rosario","America/Virgin","America/Cordoba","America/Mendoza","America/Knox_IN","America/Indianapolis","America/Shiprock","America/Catamarca","America/Argentina/ComodRivadavia","America/Atka","America/Coral_Harbour","America/Fort_Wayne","America/Ensenada","America/Porto_Acre","America/Louisville","Chile/Continental","Chile/EasterIsland","GMT0","UCT","Cuba","Hongkong","Atlantic/Faeroe","Atlantic/Jan_Mayen","GMT","Eire","US/Arizona","US/Pacific","US/Central","US/Alaska","US/Eastern","US/Samoa","US/East-Indiana","US/Mountain","US/Aleutian","US/Indiana-Starke","US/Michigan","US/Hawaii","Iran","Greenwich","NZ-CHAT","Antarctica/South_Pole","Asia/Katmandu","Asia/Kashgar","Asia/Calcutta","Asia/Thimbu","Asia/Ashkhabad","Asia/Dacca","Asia/Saigon","Asia/Tel_Aviv","Asia/Chungking","Asia/Ulan_Bator","Asia/Ujung_Pandang","Asia/Harbin","Asia/Macao","Asia/Chongqing","GMT-0","GB"); if (grep {$_ eq $CLUSTER_INFO{'Timezone'}} @Timezone) { fail("$CLUSTER_INFO{'Timezone'} timezone found on this cluster, an upgrade to OneFS 8.0.0.0 should not be performed."); } } # Output Check result info("Refer to KB502177 (https://www.dell.com/support/kbdoc/502177) for details.") if isResults(&WARN); printResults($CLUSTER_INFO{'Timezone'} or "Unknown"); } # End checkTimeZone $CHECKS{'checkUIDGID'}{'Description'} = "Checks for UID/GID values greater than 262143 for files located in / and /var."; sub checkUIDGID { if (wasCalled()) { return } return unless $RPS{'Target'}; return unless isLive(); return if isCompliance(); if (isCurrentOneFS('>=8.2')) { return } printCheck("uid_gid_check"); # Gather information from cluster my $gather = gatherCluster (qq[2>&1 ${SUDO} isi_for_array "]. q[ find -x / /var -print0 ]. q[ | xargs -0 ls -lnd ]. q[ | awk '{if (\$3 > 262143 || \$4 > 262143) print}'" ]); # Gather a list of files to preserve during upgrades, used for $root_include my $preserves = gatherLocal(q[2>&1 awk -F\" '/file name/ {print $2}' /etc/mcp/sys/user_preserve_files.xml |sed -e 's/^\///']); # Versions where the uid_gid_check was introduced may incorrectly # flag some files not touched during the upgrade. my $include_false_positive = 0; if ($RPS{'Target'} =~ /^8\.0\.0\.7$|^8\.1\.0\.[34]$|^8\.1\.1\.[01]$|^8\.1\.[23]\.0$/) { $include_false_positive = 1; } # Regex used to identify files my $var_exclude = q[^/var/(run|crash|db/versions|db/samba|db/isilon-pkg|db/pkg|apache2/run|lib/likewise/run)]; my $root_include = q[^/(].join("|", @$preserves).q[)]; # Analyze information for issues. my %uid_gid_issues = (); foreach my $lnn (@LNNS) { next unless $gather->{$lnn}; foreach my $line (@{$gather->{$lnn}}) { next unless $line =~ qr[^\s*\S+\s+\S+\s+(\d+)\s+(\d+)[^/]+(/.*)$]; my ($uid, $gid, $path) = ($1, $2, $3); if ($path =~ qr[^/var] and $path !~ qr[$var_exclude]) { $uid_gid_issues{$lnn}{$path} = 1; } elsif ($path =~ qr[$root_include]) { $uid_gid_issues{$lnn}{$path} = 1; } elsif ($include_false_positive and ($uid > 2097151 or $gid > 2097151)) { $uid_gid_issues{$lnn}{$path} = 1; } } } if (%uid_gid_issues) { fail("Files were identified as having a user/group id greater than 262143 and may cause an error in the OneFS upgrade."); fail("To review the list of files identified, re-run with the additional arguments: -e --run=checkUIDGID $RPS{'Target'}") unless $RPS{'Verbose'}; fail('For details see KB 527935 [https://www.dell.com/support/kbdoc/527935 - OneFS: Upgrade assessment fails with error "uid_gid_check check failed:"]'); my $sprintf = "%5s %s"; verbose(sprintf($sprintf, "LNN", "Path")); verbose(sprintf($sprintf, "-----", "--------------------")); foreach my $lnn (@LNNS) { next unless $uid_gid_issues{$lnn}; foreach my $path (keys %{$uid_gid_issues{$lnn}}) { verbose(sprintf($sprintf, $lnn, $path)); } } } # Output Check result printResults(); } # End checkUIDGID $CHECKS{'checkUpgradeAgentPort'}{'Description'} = "Checks the port used by the isi_upgrade_agent_d daemon to ensure it is not in use by other processes"; sub checkUpgradeAgentPort { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return unsupported("Unsupported prior to OneFS 8"); } printCheck("Upgrade Agent Port"); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[ 2>&1 ${SUDO} isi_for_array ]. qq[ sockstat -P udp -p 15100 ]); } else { $gather = gatherCluster(qq[ 2>&1 grep -H -E 'USER|15100' */sockstat ]); } # Port 15100 should be used for isi_upgrade_agent_d. my $port = 15100; my %nodes_with_problematic_pids = (); # Analyze information for issues. foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { $line =~ s/^\s*//; # Header line if ($line =~ /^\s*USER/) { verbose($line); next; } # Split the line into fields my @fields = split(/\s+/, $line); # Skip if there are less than 7 fields next if scalar(@fields) < 7; # Skip unless the PROTOCOL field is udp next unless $fields[4] =~ /udp/; # Skip unless the LOCAL ADDRESS field ends with the port next unless $fields[5] =~ /:$port$/; # Add this line to the verbose output verbose($line); # Skip if the COMMAND is isi_upgrad (short for isi_upgrade_agent_d) next if $fields[1] =~ /isi_upgrad/; # Beyond this point there must be a process other than # isi_upgrade_agent_d that is on the port. # Gather for each node the problematic PID and COMMAND $nodes_with_problematic_pids{$lnn}{$fields[2]} = $fields[1]; } } # Flag issues identified fail("A process other than the isi_upgrade_agent_d is on port $port on ". "nodes: ".compressRange(keys %nodes_with_problematic_pids)) if %nodes_with_problematic_pids; foreach my $lnn (keys %nodes_with_problematic_pids) { my @pidlist = (); foreach my $pid (keys %{$nodes_with_problematic_pids{$lnn}}) { push(@pidlist, "$nodes_with_problematic_pids{$lnn}{$pid} ($pid)"); } fail("Node $lnn: ".join(", ", @pidlist)); } # Output Check result info("Refer to KB523984 (https://www.dell.com/support/kbdoc/523984) for details.") if isResults(&WARN); printResults(); } # End checkUpgradeAgentPort $CHECKS{'checkUpgrade'}{'Description'} = "Checks for in progress upgrade related issues. Warns if the isi_upgrade_d service is enabled. Fails if not in a committed state. Fails if an upgrade activity is already in progress.Checks the fs_fmt_version, an odd or zero fs_fmt_version is problematic"; sub checkUpgrade { if (wasCalled()) { return } checkProcesses(); checkJobStatus(); checkServices(); checkUpgradeAgentPort(); printCheck("Upgrade Status"); gatherUpgradeCluster(); if ($CLUSTER_INFO{'Cluster Upgrade State'} and $CLUSTER_INFO{'Cluster Upgrade State'} ne "committed") { fail("Cluster Upgrade State is currently $CLUSTER_INFO{'Cluster Upgrade State'}."); } if ($CLUSTER_INFO{'Current Upgrade Activity'} and !($CLUSTER_INFO{'Current Upgrade Activity'} eq "-" or $CLUSTER_INFO{'Current Upgrade Activity'} =~ /No activity/i)) { fail("Current Upgrade Activity is currently $CLUSTER_INFO{'Current Upgrade Activity'}"); } checkFileSystemFormatVersion(); if (isResults(&WARN)) { info("Refer to KB 335078 (https://www.dell.com/support/kbdoc/335078) for details."); if (isCurrentOneFS('<8')) { info("See also the Isilon Customer Troubleshooting Guide for upgrades from OneFS 6.5, 7.0, 7.1, or 7.2: https://support.emc.com/docu60948"); } else { info("See also the Isilon Customer Troubleshooting Guide for upgrades from OneFS 8 and later: https://support.emc.com/docu83028"); } } # Output Check result printResults(); } # End checkUpgrade $CHECKS{'checkUptime'}{'Description'} = "Checks node uptime, warning over 200 days, flags uptime ETAs"; sub checkUptime { if (wasCalled()) { return } printCheck("Node Uptime"); gatherUptime(); gatherETA202452(); gatherETA209918(); gatherETA491747(); gatherDTA526050(); # ETA 209918 if ($RPS{'ETA'}{'209918'} and $RPS{'ETA'}{'209918'}{'Message'}) { info(@{$RPS{'ETA'}{'209918'}{'Message'}}); } # ETA 202452 if ($RPS{'ETA'}{'202452'} and $RPS{'ETA'}{'202452'}{'Message'}) { info(@{$RPS{'ETA'}{'202452'}{'Message'}}); } # ETA 491747 if ($RPS{'ETA'}{'491747'} and $RPS{'ETA'}{'491747'}{'Message'}) { info(@{$RPS{'ETA'}{'491747'}{'Message'}}); } # DTA 526050 if ($RPS{'ETA'}{'526050'} and $RPS{'ETA'}{'526050'}{'Message'}) { info(@{$RPS{'ETA'}{'526050'}{'Message'}}); } my $highest_uptime = 0; foreach my $lnn (@LNNS) { verboseHeader($NODE_INFO{$lnn}{'Name'}); verbose("Gathered uptime: $NODE_INFO{$lnn}{'uptime_gathered'}") if $NODE_INFO{$lnn}{'uptime_gathered'}; next unless $NODE_INFO{$lnn}{'uptime'}; my $uptime_seconds = $NODE_INFO{$lnn}{'uptime'}; my @calculated_uptime = (); if ($uptime_seconds > 86400) { my $days = int($uptime_seconds / 86400); $uptime_seconds -= $days * 86400; push(@calculated_uptime, "$days days") } if ($uptime_seconds > 3600) { my $hours = int($uptime_seconds / 3600); $uptime_seconds -= $hours * 3600; push(@calculated_uptime, "$hours hours") } if ($uptime_seconds > 60) { my $minutes = int($uptime_seconds / 60); $uptime_seconds -= $minutes * 60; push(@calculated_uptime, "$minutes minutes") } if ($uptime_seconds > 0) { push(@calculated_uptime, "$uptime_seconds seconds") } verbose("Calculated uptime: ".join(", ", @calculated_uptime)) if @calculated_uptime; $highest_uptime = $NODE_INFO{$lnn}{'uptime'} if ($NODE_INFO{$lnn}{'uptime'} > $highest_uptime); } # Not used, but may come in handy debug("checkUptime: Predicted date for 248.5 day uptime bug: ".scalar(localtime($RPS{'Runtime'} + 21384000 - $highest_uptime))); # Warn about/provide information for node uptime and ETAs if ($highest_uptime > 42940800) { if ($RPS{'ETA'}{'202452'}) { warning("ETA202452: Nodes over 497 days uptime."); push(@{$RPS{'Proactive Reboot'}{'Priority'}}, "During any activity that requires node reboots it is possible that all remaining nodes in the cluster restart simultaneously, without warning, due to ETA 202452."); } if ($RPS{'ETA'}{'526050'}) { warning("DTA526050: Nodes over 497 days uptime."); } } elsif ($highest_uptime > 41731200) { if ($RPS{'ETA'}{'202452'}) { warning("ETA202452: Nodes approaching 497 days uptime."); } if ($RPS{'ETA'}{'526050'}) { warning("DTA526050: Nodes approaching 497 days uptime."); } } if ($highest_uptime > 21470400 and $RPS{'ETA'}{'209918'}) { warning("ETA209918: Nodes over 248.5 days uptime."); push(@{$RPS{'Proactive Reboot'}{'Priority'}}, "During any activity that requires node reboots it is possible that all remaining nodes in the cluster restart simultaneously, without warning, due to ETA 209918."); } elsif ($highest_uptime > 20217600 and $RPS{'ETA'}{'209918'}) { warning("ETA209918: Nodes approaching 248.5 days uptime."); } if ($highest_uptime > 21470400 and $RPS{'ETA'}{'491747'}) { warning("ETA491747: Nodes over 248.5 days uptime."); } elsif ($highest_uptime > 20217600 and $RPS{'ETA'}{'491747'}) { warning("ETA491747: Nodes approaching 248.5 days uptime."); } # Proactive reboot only for versions of OneFS prior to 8.0 if (isCurrentOneFS('<8') and $highest_uptime > 17280000) { push(@{$RPS{'Proactive Reboot'}{'Priority'}}, "A proactive reboot preceding all other activities is strongly recommended due to the high uptime of nodes."); warning("Highest node uptime over 200 days.") if !isResults(&WARN); @{$RPS{'Proactive Reboot'}{'Nodes'}} = keys %NODE_INFO; } info("Refer to KB 335097 (https://www.dell.com/support/kbdoc/335097) for details") if isResults(&WARN); my $highest_uptime_days = int($highest_uptime / 86400); printResults("$highest_uptime_days days"); } # End checkUptime $CHECKS{'checkZoneLocalAuth'}{'Description'} = 'For upgrades to OneFS 8.2 and later, checks'. ' if local providers are associated with other access zones'; sub checkZoneLocalAuth { if (wasCalled()) { return } if (isCurrentOneFS('<8') or isCurrentOneFS('>=8.2')) { return unsupported('Supported in OneFS 8 and OneFS 8.1'); } if (!isDestinationOneFS('>=8.2')) { return unsupported('Supported when target_version parameter is OneFS'. ' 8.2 and newer'); } # Setup my $NAME = 'Access Zone Local Authentication Providers'; my $GREP = 'egrep "^ *(Name|Auth Providers):"'; my $CMD_LIVE = qq[${SUDO} isi zone zones list --verbose | $GREP]; my $CMD_LOGS = qq[$GREP local/isi_auth | grep -B1000 "Auth Providers"]; my $MSG_WARN = 'Warning: local providers (with the exception of System'. ' local provider) may no longer be added to other access zones'. ' (beyond the one that shares their name) after upgrade to OneFS'. ' 8.2.0+. Local providers from the list below that are currently'. ' added to other access zones will continue to work; however, if they'. ' are removed, they will not be permitted to be re-added. Please use'. ' only the System local provider to share local users between access'. ' zones.'; my $MSG_KB = 'For additional details review KB 000165335'. ' [https://www.dell.com/support/kbdoc/en-us/000165335]'; my $SPF_MISMATCH = 'Zone: %s, Providers: %s'; printCheck($NAME); # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $zone_name = ''; my %mismatch = (); foreach my $line (@$gather) { if ($line =~ /^\s*Name:\s*(\S+)$/) { $zone_name = $1 } if (!$zone_name) { next } if ($line !~ /^\s*Auth Providers:\s*(\S.*)\s*$/) { next } my @providers = split(/, /, $1); foreach my $provider (@providers) { my ($type, $zone) = split(/:/, $provider); if ($zone eq 'System' or $zone eq $zone_name) { next } if ($type !~ /local/) { next } push(@{$mismatch{$zone_name}}, $provider); } } if (%mismatch) { addCustomerInformation($NAME, warning($MSG_WARN)); foreach my $zone_name (keys %mismatch) { my $msg = sprintf($SPF_MISMATCH, $zone_name, join(', ', @{$mismatch{$zone_name}})); addCustomerInformation($NAME, warning($msg)); } info($MSG_KB); } printResults(); } # End checkZoneLocalAuth ### # Checks for the existence of additional certificates in # /ifs/.ifsvar/modules/cloud/cacert sub checkCloudPoolsCerts { if (wasCalled()) { return } my $PATH_LOGS = q[local/ifsvar_modules.tar/modules/cloud/cacert]; my $CMD_LOGS = qq[2>&1 find $PATH_LOGS -type f]; my $CMD_LIVE = q[2>&1 find /ifs/.ifsvar/modules/cloud/cacert -type f]; my $MSG_CERTS = 'Found additional certificates under'. ' /ifs/.ifsvar/modules/cloud/cacert/. If you are using SSL/TLS and a'. ' certificate that required installation of a root / intermediate'. ' certificate previously, the certificate will not be automatically'. ' migrated to the new certificate store after upgrading to OneFS 8.2'. ' or later. Please make a note to perform this certificate import'. ' post upgrade. See KB 491608'. ' [https://www.dell.com/support/kbdoc/491608 - CloudPool Account'. ' Create Failed or CloudPool account is unreachable with - clapi'. ' error: CL_SSL_CACERT]'; my @DEFAULT_CERTS = ( 'BaltimoreCyberTrustRoot.pem', 'DigiCertHighAssuranceEV-RootCA.pem', 'DigiCertSHA2HighAssuranceServerCA.pem', 'GeoTrust_Global_CA.pem', 'GoogleInternetAuthority_G2.pem', 'GoogleInternetAuthority_G3.pem', 'VerisignClass3PublicPrimaryCertificationAuthority.pem', ); # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze my @certs = (); foreach my $line (@$gather) { # Skip filenames that match the default certificates my ($is_default) = grep {$line =~ /$_$/i} @DEFAULT_CERTS; if ($is_default) { next } # Check file extension for a non-default certificate if ($line =~ /\/([^\/]+\.pem)$/i) { push(@certs, $line) } } # Flag if (@certs) { addCustomerInformation('CloudPools', critical($MSG_CERTS)); verboseHeader('CloudPool certificates'); verbose($_) foreach @certs; } } # End checkCloudPoolsCerts ### # checkCloudPoolsCOIDump: Checks for active Cloud Object Index dumps and # existing dump files sub checkCloudPoolsCOIDump { if (wasCalled()) { return } if (!$RPS{'Live'}) { return } my $CMD_COIDUMPS = q[2>&1 find /ifs/data/Isilon_Support/ -type f -iname]. q[ '*coi_dump*']; my $CMD_NODE = q[screen -ls | grep -i coi_dump]; my $CMD_COISCREENS = qq[2>&1 ${SUDO} isi_for_array '$CMD_NODE']; my $SPF_SCREEN = 'A coi_dump screen session was detected on the following'. ' node(s): %s'; my $MSG_COIDUMPS = 'Multiple COI dumps identified under'. ' /ifs/data/Isilon_Support. Re-run IOCA with --run=checkCloudPools'. ' --extra for a list of the COI dump(s) identified.'; my $SPF_COIDUMP = 'COI dump identifed: %s'; my $SPF_VERBOSE = '%5s %s'; # Check for coi dump files in /ifs/data/Isilon_Support my $gather = gatherLocal($CMD_COIDUMPS); if (@$gather) { if (scalar(@$gather) == 1) { details(sprintf($SPF_COIDUMP, $gather->[0])); } elsif (scalar(@$gather) > 1) { details($MSG_COIDUMPS); } verboseHeader('Cloud Object Index (COI) dump files'); verbose(@$gather); } # Check for coi_dump screen sessions $gather = gatherCluster($CMD_COISCREENS); if (%$gather) { warning(sprintf($SPF_SCREEN, compressRange(keys %$gather))); verboseHeader('Cloud Object Index (COI) dump screen sessions'); verbose(sprintf($SPF_VERBOSE, 'LNN', 'Screen session')); verbose(sprintf($SPF_VERBOSE, '-'x5, '-'x20)); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { s/^\s+|\s+$//g; verbose(sprintf($SPF_VERBOSE, $lnn, $_)); } } } } # End checkCloudPoolsCOIDump sub checkCloudPoolsNANON { if (wasCalled()) { return } if (!$RPS{'Live'}) { fail(q[Verification of nodes that do not have connectivity to cloud]. q[ providers cannot be completed via logs and must be checked]. q[ on cluster with the following command: isi_for_array 'isi]. q[ cloud account list' | egrep -vi '(ok|disabled) *$']); $RPS{'Requires TC'} = 1; return; } my $CMD_NODE = qq[${SUDO} isi cloud account list -az --format=csv]; my $CMD_GREP = q[egrep -vi '(ok|disabled) *$']; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array '$CMD_NODE'|$CMD_GREP]; my $SPF_FOUND = 'The following nodes may not be able to communicate with'. ' all CloudPools providers: %s. Nodes that do not have connectivity to'. ' cloud providers will delay or block ability to upgrade stub files in'. ' snapshots and may cause the SnapshotDelete job to fail when trying'. ' to process file fill requests. This requires each node to have cloud'. ' provider connectivity. If this cannot be accomplished reach out to'. ' Isilon Support for assistance prior to performing the upgrade.'; my $SPF_VERBOSE = '%5s %s'; # Gather my $gather = gatherCluster($CMD_LIVE); # Analyze my %nodes = (); foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach my $line (@{$gather->{$lnn}}) { if ($line !~ /,/) { next } my ($account) = split /,/, $line; push(@{$nodes{$lnn}}, $account); } } # Flag if (%nodes) { my $msg = sprintf($SPF_FOUND, compressRange(keys %nodes)); addCustomerInformation('CloudPools', fail($msg)); verboseHeader('CloudPool account(s) with connectivity issues by node'); verbose(sprintf($SPF_VERBOSE, 'LNN', 'CloudPools account(s)')); verbose(sprintf($SPF_VERBOSE, '-'x5, '-'x20)); foreach my $lnn (@LNNS) { if (!$nodes{$lnn}) { next } verbose(sprintf($SPF_VERBOSE, $lnn, join(',', @{$nodes{$lnn}}))); } } } # End checkCloudPoolsNANON ### # checkCloudPoolsSmartLink: Checks if there is a running smartlink-upgrade job sub checkCloudPoolsSmartLink { if (wasCalled()) { return } if (!$RPS{'Live'}) { return } if (isCurrentOneFS('<8.2')) { return unsupported('Supported in OneFS 8.2 and later'); } # Setup my $MSG_SMARTLINK = 'CloudPools smartlink-upgrade job is currently running'. ' and may be monitored with the following command:'. ' isi cloud jobs view 6'; my $CMD_LIVE = qq[2>&1 ${SUDO} isi cloud jobs view 6]; # Gather my $gather = gatherLocal($CMD_LIVE); # Analyze my $smartlink_upgrade_remaining_files = 0; my $job_id = 0; foreach (@$gather) { if (/^\s*ID:\s+(\d+)$/) { $job_id = $1 } if ($job_id ne 6) { next } if (/^\s*Total\s(?:Pending|Processing):\s+(\d+)$/) { $smartlink_upgrade_remaining_files += $1; } } # Flag if ($smartlink_upgrade_remaining_files) { warning($MSG_SMARTLINK) } } # End checkCloudPoolsSmartLink ### # checkCloudPoolsSSD: Checks SSD usage for cloudpools conversion sub checkCloudPoolsSSD { if (wasCalled()) { return } if (isCurrentOneFS('>=8.2')) { return } if (!isDestinationOneFS('>=8.2')) { return } if (!hasCloudPools()) { return } # Setup my $THRESHOLD_SSD_USED = 60; my $MSG_SSD_ASSESSMENT = ''; # Gather gatherStatus(); gatherETA518924(); # Analyze # No SSD Storage or configured for L3 if (defined $CLUSTER_INFO{'SSD Size'} and $CLUSTER_INFO{'SSD Size'} == 0) { return; } # SSD usage less than 60% threshold if (defined $CLUSTER_INFO{'SSD Percent Used'} and $CLUSTER_INFO{'SSD Percent Used'} <= $THRESHOLD_SSD_USED) { return; } # Check for recent Collect/MultiScan job runs on versions not at # risk for ETA518924 if (!defined $RPS{'ETA'}{'518924'}{'Severity'}) { } #TBD # Flag fail($MSG_SSD_ASSESSMENT); } # End checkCloudPoolsSSD $CHECKS{'checkCloudPools'}{'Description'} = 'Checks for CloudPools related issues'; sub checkCloudPools { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return unsupported('Requires OneFS 8 and later'); } if (!hasLicense('CloudPools')) { return unsupported('Requires CloudPools license'); } checkSyncIQ(); printCheck('CloudPools'); my $MSG_CPKB = 'When using CloudPools, there are additional steps required'. ' by customer alongside an upgrade to OneFS 8.2.2 and later. For'. ' details review KB 544023 [https://www.dell.com/support/kbdoc/544023'. ' - Isilon Cloudpools - Upgrading 8.x to 8.2.2]'; my $MSG_SYNCIQ = 'It is strongly recommended that the OneFS upgrade to be'. ' committed and the CloudPools conversion process started on Target'. ' cluster before committing the OneFS upgrade on the Source cluster.'. ' This allows SyncIQ to send CP1.0 formatted stub files from the'. ' Source cluster to the Target cluster, where they will be converted'. ' into CP2.0 formatted stub files.'; my $SPF_CP_LOW_DEST = 'The provided destination version, OneFS %s, is lower'. ' than the minimum required when using CloudPools, OneFS 8.2.2 or later.'; if (hasCloudPools()) { checkCloudPoolsCOIDump(); checkCloudPoolsSmartLink(); if ($RPS{'Target'} and isCurrentOneFS('<8.2') and compareVersions($RPS{'Target'}, '8.2') >= 0) { if (compareVersions($RPS{'Target'}, '8.2.2') < 0) { fail(sprintf($SPF_CP_LOW_DEST, $RPS{'Target'})); } addCustomerInformation('CloudPools', warning($MSG_CPKB)); checkCloudPoolsCerts(); checkCloudPoolsNANON(); if ($RPS{'SyncIQ'}) { addCustomerInformation('CloudPools', $MSG_SYNCIQ); } } } printResults(); } # End checkCloudPools $CHECKS{'checkKB496993'}{'Description'} = 'Checks if the cluster is at risk for KB496993'; sub checkKB496993 { if (wasCalled()) { return } printCheck('KB 496993'); my $MSG_DU = 'CloudPools is enabled but the isi_cpool_d and/or isi_cpool_io_d service is disabled and may cause Data Unavailability per KB 496993'; my $MSG_KB = 'For additional details review KB 496993 [https://www.dell.com/support/kbdoc/496993 - Isilon OneFS: OneFS 8.x and later : Cloud Pools Services must be enabled if it is licensed as it can cause a DU for SMB/NFS]'; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_gconfig registry.Services.lwsm.Parameters.Variables.CloudpoolsEnabled]; my $CMD_LOGS = qq[2>&1 grep registry.Services.lwsm.Parameters.Variables.CloudpoolsEnabled local/isi_gconfig]; # If running OneFS versions prior to 8.0, # flag the issue for an upgrade to 8 or later if (isCurrentOneFS('<8')) { if ($RPS{'Target'} and compareVersions($RPS{'Target'}, '8') >= 0 and hasLicense('CloudPools') and (isServiceDisabled('isi_cpool_d') or isServiceDisabled('isi_cpool_io_d'))) { fail($MSG_DU); info($MSG_KB); } printResults(); return; } # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze my $cloudpools_enabled = 0; foreach (@$gather) { my ($key, $value) = parseGconfigItem($_); if ($key =~ /CloudpoolsEnabled/) { $cloudpools_enabled = $value; } } # Flag if ($cloudpools_enabled and (isServiceDisabled('isi_cpool_d') or isServiceDisabled('isi_cpool_io_d'))) { fail($MSG_DU); info($MSG_KB); } # Output printResults(); } # End checkKB496993 ### # hasCloudPools: Used to determine CloudPools usage sub hasCloudPools { if (defined $CLUSTER_INFO{'hasCloudPools'}) { return $CLUSTER_INFO{'hasCloudPools'}; } $CLUSTER_INFO{'hasCloudPools'} = 0; if (!hasLicense('CloudPools')) { return $CLUSTER_INFO{'hasCloudPools'}; } my $CMD_GREP = q[egrep '\.(provider_instance_id|account_id) ']; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_gconfig -t cpool_cfg | $CMD_GREP]; my $CMD_LOGS = qq[2>&1 sqlite3 local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/Gconfig.cpool_config.sqlite 'select * from kv_table' | $CMD_GREP]; my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } my $has_cloud_account = 0; my $has_cloud_provider = 0; foreach (@$gather) { if (/^provider_instance\.\d+\.provider_instance_id/) { $has_cloud_provider = 1; } if (/^account\.\d+\.account_id/) { $has_cloud_account = 1; } } if ($has_cloud_account and $has_cloud_provider) { $CLUSTER_INFO{'hasCloudPools'} = 1; } return $CLUSTER_INFO{'hasCloudPools'}; } # End hasCloudPools ### # gatherArrayXML: Gathers information from /etc/ifs/array.xml sub gatherArrayXML { if (wasCalled()) { return } # A bug may cause the array.xml files to be inconsistent across nodes # So, revert to gathering committed_version from each node my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array]. q[ 'egrep "<(committed_version|fileserial)>" /etc/ifs/array.xml']; my $CMD_LOGS = q[2>&1 egrep -H "<(committed_version|fileserial)>"]. q[ */etc.tar/etc/ifs/array.xml */etcifs.tar/ifs/array.xml]; my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); if (!%$gather) { return } foreach my $lnn (keys %$gather) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/<(committed_version|fileserial)>([^<]+)${lnn}->${1}=${2}"); } } } # Gather my $grep = '&1 grep -h -E '$grep' /etc/ifs/array.xml]); } else { my ($file) = gatherLocal(q[2>/dev/null ls -t]. q[ */etc.tar/etc/ifs/array.xml]. q[ */etcifs.tar/ifs/array.xml|head -1]); if (-e $file) { $gather = gatherLocal(qq[2>&1 grep -h -E '$grep' '$file']); } } # Analyze my %interface = (); my %device = (); my $gather_array = 0; my $gather_interfaces = 0; foreach (@$gather) { if (//) { $gather_array = 1 } if (/<\/array>/) { $gather_array = 0 } if (//) { $gather_interfaces = 1 } if (/<\/interfaces>/) { $gather_interfaces = 0 } if ($gather_interfaces) { # Gather interface details if (/(name)=\"([^\"]+)\"/) { $interface{$1} = $2 } if (/(flags)=\"([^\"]+)\"/) { $interface{$1} = $2 } if (/<(netmask|low|high|mtu)>([^<]+)/) { if ($interface{'name'}) { my $name = $interface{'name'}; if ($interface{'flags'}) { $CLUSTER_INFO{'Interfaces'}{$name}{'Flags'} = $interface{'flags'}; debug("CLUSTER_INFO->Interfaces->${name}->Flags=". $interface{'flags'}); } if ($interface{'netmask'}) { $CLUSTER_INFO{'Interfaces'}{$name}{'Netmask'} = $interface{'netmask'}; debug("CLUSTER_INFO->Interfaces->${name}->Netmask=". $interface{'netmask'}); } if ($interface{'low'}) { $CLUSTER_INFO{'Interfaces'}{$name}{'Low'} = $interface{'low'}; debug("CLUSTER_INFO->Interfaces->${name}->Low=". $interface{'low'}); } if ($interface{'high'}) { $CLUSTER_INFO{'Interfaces'}{$name}{'High'} = $interface{'high'}; debug("CLUSTER_INFO->Interfaces->${name}->High=". $interface{'high'}); } if ($interface{'mtu'}) { $CLUSTER_INFO{'Interfaces'}{$name}{'MTU'} = $interface{'mtu'}; debug("CLUSTER_INFO->Interfaces->${name}->MTU=". $interface{'mtu'}); } } %interface = (); } } elsif ($gather_array) { # Gather device details if (/<(array_lnn|array_id|type|onefs_version)>([^<]+)/) { if ($device{'array_lnn'}) { my $lnn = $device{'array_lnn'}; if ($CLUSTER_INFO{'Name'}) { $NODE_INFO{$lnn}{'Name'} = "$CLUSTER_INFO{'Name'}-$lnn"; debug("NODE_INFO->${lnn}->Name=". $NODE_INFO{$lnn}{'Name'}); } if ($device{'type'}) { $NODE_INFO{$lnn}{'type'} = $device{'type'}; debug("NODE_INFO->${lnn}->type=$device{'type'}"); } if ($device{'onefs_version'}) { $NODE_INFO{$lnn}{'onefs_version'} = $device{'onefs_version'}; debug("NODE_INFO->${lnn}->onefs_version=". $device{'onefs_version'}); } if ($device{'array_id'}) { $NODE_INFO{$lnn}{'ID'} = $device{'array_id'}; debug("NODE_INFO->${lnn}->ID=$NODE_INFO{$lnn}{'ID'}"); $LNN_FROM_DEVID{$device{'array_id'}} = $lnn; debug("LNN_FROM_DEVID->${device{'array_id'}}=$lnn"); } } %device = (); } } else { # Gather cluster details if (/([^<]+)Name=$1"); } if (/([^<]+)Encoding=$1"); } if (/([^<]+)Timezone=$1"); } if (/([^<]+)GUID=$1"); } if (/([^<]+)Compliance=$1"); } } } # Populate the number of nodes $CLUSTER_INFO{'Node Count'} = scalar(keys %NODE_INFO); debug("CLUSTER_INFO->Node Count=$CLUSTER_INFO{'Node Count'}"); # Create a sorted list of logical node numbers [lnns] @LNNS = sort {$a <=> $b} keys %NODE_INFO; debug("Sorted lnn list: [@LNNS]"); } # End gatherArrayXML sub gatherEventChannel { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return } my $MATCH = q[ID|Name|Type|Enabled]; my $CMD_GREP = qq[ egrep '$MATCH' ]; my $CMD_LOGS = qq[2>&1 $CMD_GREP local/isi_event_channel]; my $CMD_LIVE = qq[2>&1 ${SUDO} isi event channel list -v | $CMD_GREP]; # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Store relavant information my $id = q[]; foreach my $line (@$gather) { # Skip unless the line is a match if ($line !~ /^\s*(?:$MATCH)/) { next } # Remove any leading whitespaces $line =~ s/^\s*//; # Split into a name/value pair my ($name, $value) = split(/:\s*/, $line, 2); # Assign the ID if ($name eq 'ID') { $id = $value } # Skip if no ID is found if (!$id) { next } # Assign values $CLUSTER_INFO{'Event Channels'}{$id}{$name} = $value; debug("CLUSTER_INFO->Event Channels->${id}->${name}=$value"); } } # End gatherEventChannel ### # gatherEventGroups8: Gathers events on OneFS 8+ clusters. sub gatherEventGroups8 { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { debug("gatherEventGroups8: Skipping. OneFS version prior to 8."); return; } # Gather event group information my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi event groups list| grep critical | grep '\\-\\-'"); } else { $gather = gatherLocal("2>&1 grep -h critical */isi_event_groups | grep '\\-\\-'"); } foreach my $line (@$gather) { next if ($line =~ /^ID/ or $line =~ /^-/ or $line =~ /^Total/); next unless ($line =~ /^(\d+)\s+([\d\/]+\s+[\d:]+)\s+(--)\s+(\S+)\s+(\d+)\s+(\d+)\s+(critical)$/); my ($id, $started, $ended, $causes, $lnn, $events, $severity) = ($1, $2, $3, $4, $5, $6, $7); # Assign out variables $CLUSTER_INFO{'Events'}{$id}{'Started'} = $started; debug("CLUSTER_INFO->Events->${id}->Started=$started"); $CLUSTER_INFO{'Events'}{$id}{'Ended'} = $ended; debug("CLUSTER_INFO->Events->${id}->Ended=$ended"); $CLUSTER_INFO{'Events'}{$id}{'Severity'} = $severity; debug("CLUSTER_INFO->Events->${id}->Severity=$severity"); } } # End gatherEventGroups8 ### # gatherEvents7: Gathers events on OneFS 7 and earlier clusters. sub gatherEvents7 { if (wasCalled()) { return } if (isCurrentOneFS('>=8')) { debug("gatherEvents7: Skipping for OneFS versions 8 and later."); return; } # Gather event group information my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi events list -w | grep '\\-\\- .* C ']); } else { $gather = gatherLocal(qq[2>&1 grep -h '\\-\\- .* C ' */isi_alerts */isi_events | sort | uniq]); } # Iterate through gather foreach my $line (@$gather) { next unless ($line =~ /^([\d\.]+)\s+([\d\/]+\s+[\d:]+)\s+(--)\s+(C)\s+(\S+)\s+(.*?)$/); my ($id, $started, $ended, $severity, $lnn, $message) = ($1, $2, $3, $4, $5, $6); # Skip event if already recorded next if $CLUSTER_INFO{'Events'}{$id}; # Assign out variables $CLUSTER_INFO{'Events'}{$id}{'Started'} = $started; debug("CLUSTER_INFO->Events->${id}->Started=$started"); $CLUSTER_INFO{'Events'}{$id}{'Ended'} = $ended; debug("CLUSTER_INFO->Events->${id}->Ended=$ended"); $CLUSTER_INFO{'Events'}{$id}{'Severity'} = $severity; debug("CLUSTER_INFO->Events->${id}->Severity=$severity"); push(@{$CLUSTER_INFO{'Events'}{$id}{'Lnn'}}, $lnn); debug("CLUSTER_INFO->Events->${id}->Lnn=$lnn"); push(@{$CLUSTER_INFO{'Events'}{$id}{'Message'}}, $message); debug("CLUSTER_INFO->Events->${id}->Message=$message"); } } # End gatherEvents7 ### # gatherEvents8: Gathers events on OneFS 8+ clusters. sub gatherEvents8 { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { debug("gatherEvents8: Skipping. OneFS version prior to 8."); return; } gatherEventGroups8(); return unless $CLUSTER_INFO{'Events'}; # Gather event information my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi event events list --severity=critical|grep '".join("\\|", keys %{$CLUSTER_INFO{'Events'}})."'") if $CLUSTER_INFO{'Events'}; } else { $gather = gatherLocal("2>&1 sqlite3 local/celog_ifsvar_db.tar/ifs/_ifsvar/db/celog/events.db 'select devid, coalesce_instanceid, message from events where coalesce_instanceid in (".join(",", keys %{$CLUSTER_INFO{'Events'}}).")'") if $CLUSTER_INFO{'Events'}; } # Iterate through gather foreach my $line (@$gather) { next if ($line =~ /^ID/ or $line =~ /^Total/ or $line =~ /^-/); my ($lnn, $eventgroup, $message) = ""; if ($line =~ /^(\d+)\|(\d+)\|(.*)$/) { ($lnn, $eventgroup, $message) = ($1, $2, $3); $lnn = $LNN_FROM_DEVID{$1} if $LNN_FROM_DEVID{$1}; } elsif ($line =~ /^\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+(\S+)\s+(.*)$/) { ($lnn, $eventgroup, $message) = ($1, $2, $3); } $lnn = "All" if defined $lnn and $lnn eq "0"; next unless $lnn and $eventgroup and $message; # Assign out variables my @results = grep {$_ eq $lnn or $_ eq "All"} @{$CLUSTER_INFO{'Events'}{$eventgroup}{'Lnn'}}; unless (@results) { push(@{$CLUSTER_INFO{'Events'}{$eventgroup}{'Lnn'}}, $lnn); debug("CLUSTER_INFO->Events->${eventgroup}->Lnn=$lnn"); } @results = grep {$_ eq $message} @{$CLUSTER_INFO{'Events'}{$eventgroup}{'Message'}}; unless (@results) { push(@{$CLUSTER_INFO{'Events'}{$eventgroup}{'Message'}}, $message); debug("CLUSTER_INFO->Events->${eventgroup}->Message=$message"); } } } # End gatherEvents8 ### # gatherEvents: Gathers events sub gatherEvents { return if wasCalled(); gatherEvents7(); gatherEvents8(); } # End gatherEvents sub gatherIsiStatPools { if (wasCalled()) { return } # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { if (isCurrentOneFS('<8')) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi stat -d -v]); } else { $gather = gatherLocal(qq[2>&1 ${SUDO} isi --timeout=300 stat -p -v]); } } else { $gather = gatherLocal(qq[2>&1 cat */isi_stat-d */isi_stat-p]); } # Analyze information for issues. my $node_group_name = ""; my $seen_cluster_name = 0; foreach my $line (@$gather) { # Gather cluster name if ($line =~ /^Cluster Name:\s+(\S+)$/) { last if $seen_cluster_name; $CLUSTER_INFO{'Name'} = $1; $seen_cluster_name = 1; debug("CLUSTER_INFO->Name=$CLUSTER_INFO{'Name'}"); } # Gather cluster health if ($line =~ /^Cluster Health:\s+\[\s*(\S+)\s*\]$/) { if (!$CLUSTER_INFO{'Health'}) { $CLUSTER_INFO{'Health'} = $1; debug("CLUSTER_INFO->Healthy=$CLUSTER_INFO{'Health'}"); } } # Start gathering pool information if ($line =~ /^(?:Node|Disk) (?:Group|Pool) Name:\s+(\S+)(?:\s+Protection:\s+(\S+))?/) { $node_group_name = $1; if (defined $2) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'Protection'} = $2; debug("CLUSTER_INFO->Pools->${node_group_name}->Protection=$2"); } } # Only proceed from this point if we have gathered a group name next unless $node_group_name; # Gather VHS details for the group if ($line =~ /^VHS Size:\s+(\S+)/) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'VHS'} = $1; debug("CLUSTER_INFO->Pools->${node_group_name}->VHS=$1"); } # Gather %Used details for the group if ($line =~ /^Used:.*?(\d+)%(?:.*?(\d+)%)?/) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'Percent Used'} = $1; debug("CLUSTER_INFO->Pools->${node_group_name}->Percent Used=$1"); if (defined $2) { $CLUSTER_INFO{'Pools'}{$node_group_name}{'Percent HDD Used'} = $1; debug("CLUSTER_INFO->Pools->${node_group_name}->Percent HDD Used=$1"); $CLUSTER_INFO{'Pools'}{$node_group_name}{'Percent SSD Used'} = $2; debug("CLUSTER_INFO->Pools->${node_group_name}->Percent SSD Used=$2"); } } next if ($line =~ /^Name/); if ($line =~ /^\S+\s*\|/) { my @columns = split(/\s*\|\s*/, $line); $CLUSTER_INFO{'Pools'}{$node_group_name}{'Health'} = $columns[1]; debug("CLUSTER_INFO->Pools->${node_group_name}->Health=$columns[1]"); $CLUSTER_INFO{'Pools'}{$node_group_name}{'HDD Storage'} = $columns[5]; debug("CLUSTER_INFO->Pools->${node_group_name}->HDD Storage=$columns[5]"); $CLUSTER_INFO{'Pools'}{$node_group_name}{'SSD Storage'} = $columns[$#columns]; debug("CLUSTER_INFO->Pools->${node_group_name}->SSD Storage=$columns[$#columns]"); } next unless ($line =~ /^\s*(\d+)\|/); my $lnn = $1; push(@{$CLUSTER_INFO{'Pools'}{$node_group_name}{'Members'}}, $lnn); debug("CLUSTER_INFO->Pools->${node_group_name}->Members=$lnn"); $NODE_INFO{$lnn}{'Pool'} = $node_group_name; debug("NODE_INFO->${lnn}->Pool=$node_group_name"); my @columns = split(/\s*\|\s*/, $line); $NODE_INFO{$lnn}{'Health'} = $columns[2]; debug("NODE_INFO->${lnn}->Health=$NODE_INFO{$lnn}{'Health'}"); $NODE_INFO{$lnn}{'HDD Storage'} = $columns[6]; debug("NODE_INFO->${lnn}->HDD Storage=$NODE_INFO{$lnn}{'HDD Storage'}"); $NODE_INFO{$lnn}{'SSD Storage'} = $columns[$#columns]; debug("NODE_INFO->${lnn}->SSD Storage=$NODE_INFO{$lnn}{'SSD Storage'}"); if ($NODE_INFO{$lnn}{'HDD Storage'} =~ /(\d+)%/ or $NODE_INFO{$lnn}{'SSD Storage'} =~ /(\d+)%/) { $NODE_INFO{$lnn}{'Percent Used'} = $1; debug("NODE_INFO->${lnn}->Percent Used=$1"); } if ($NODE_INFO{$lnn}{'SSD Storage'} =~ /L3/) { $NODE_INFO{$lnn}{'L3'} = 1; debug("NODE_INFO->${lnn}->L3=1"); } } } # End gatherIsiStatPools ### # gatherLicense7: Gather and store license details sub gatherLicense7 { if (wasCalled()) { return } if (isCurrentOneFS('>=8')) { return } my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi license"); } else { $gather = gatherLocal("2>&1 cat local/isi_license"); } # Analyze information for issues. my $unpack_template = ""; my $min_length = 0; foreach my $line (@$gather) { # Generate an unpack template from the header line if ($line =~ /^Module/) { $min_length = length($line); $unpack_template = generateUnpackTemplate($line, "Module", "License", "Configuration", "Expiration"); } # Skip header/foot lines and skip until an unpack templace is available next if $line =~ /^\s*-/; next if $line =~ /^Module/; next unless $unpack_template; # Expand the line to be at least as long as the header line to avoid unpack issues $line = sprintf("%-${min_length}s", $line); # Unpack the line my @unpacked_line = (); eval { @unpacked_line = unpack($unpack_template, $line); }; $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Status'} = $unpacked_line[1]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Status=$unpacked_line[1]"); if ($unpacked_line[3]) { $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Expiration'} = $unpacked_line[3]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Expiration=$unpacked_line[3]"); } } } # End gatherLicense7 ### # gatherLicense80: Gather and store license details sub gatherLicense80 { if (wasCalled()) { return } if (!isCurrentOneFS(/^8\.0/)) { return } my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi license list"); } else { $gather = gatherLocal("2>&1 cat local/isi_license"); } # Analyze information for issues. my $unpack_template = ""; my $min_length = 0; foreach my $line (@$gather) { # Generate an unpack template from the header line if ($line =~ /^Name/) { $min_length = length($line); $unpack_template = generateUnpackTemplate($line, "Name", "Status", "Expiration"); } # Skip header/foot lines and skip until an unpack templace is available next if $line =~ /^\s*-/; next if $line =~ /^Total/; next if $line =~ /^Name/; next unless $unpack_template; # Expand the line to be at least as long as the header line to avoid unpack issues $line = sprintf("%-${min_length}s", $line); # Unpack the line my @unpacked_line = (); eval { @unpacked_line = unpack($unpack_template, $line); }; $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Status'} = $unpacked_line[1]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Status=$unpacked_line[1]"); if ($unpacked_line[2] ne "-") { $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Expiration'} = $unpacked_line[2]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Expiration=$unpacked_line[2]"); } } } # End gatherLicense80 ### # gatherLicense81: Gather and store license details sub gatherLicense81 { if (wasCalled()) { return } if (isCurrentOneFS('<8.1')) { return } my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi license list"); } else { $gather = gatherLocal("2>&1 cat local/isi_license"); } # Analyze information for issues. my $unpack_template = ""; my $min_length = 0; foreach my $line (@$gather) { # Generate an unpack template from the header line if ($line =~ /^Module/) { $min_length = length($line); $unpack_template = generateUnpackTemplate($line, "Module", "Licensed", "Actual", "Status", "Expiration"); } # Gather OneFS License ID if ($line =~ /OneFS License ID:\s*(\S+)$/ and $1 ne "-") { $CLUSTER_INFO{'OneFS License ID'} = $1; debug("CLUSTER_INFO->OneFS License ID=$1"); } # Gather if cluster has a "Valid Signature" if ($line =~ /Valid Signature:\s*Yes$/) { $CLUSTER_INFO{'Valid Signature'} = "Yes"; debug("CLUSTER_INFO->Valid Signature=Yes"); } # Skip header/foot lines and skip until an unpack templace is available next if $line =~ /^[\s-]*$/; next if $line =~ /^Total/; next if $line =~ /^Module/; next unless $unpack_template; # Unpack the line eval { # Expand the line to be at least as long as the header line to avoid unpack issues $line = sprintf("%-${min_length}s", $line); my @unpacked_line = (); @unpacked_line = unpack($unpack_template, $line); # Assign values out $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Status'} = $unpacked_line[3]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Status=$unpacked_line[3]"); if ($unpacked_line[4] ne "-") { $CLUSTER_INFO{'License'}{$unpacked_line[0]}{'Expiration'} = $unpacked_line[4]; debug("CLUSTER_INFO->License->${unpacked_line[0]}->Expiration=$unpacked_line[4]"); } }; } } # End gatherLicense81 ### # gatherLicense: Gather and store license details sub gatherLicense { gatherLicense7(); gatherLicense80(); gatherLicense81(); } # End gatherLicense ### # gatherNeighborhoods: Gather neighborhoods from disk pool information sub gatherNeighborhoods { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return } # Setup my @COLS = ('Name', 'Id', 'Type', 'Prot', 'Flags', 'Members', 'VHS', 'HDD Used', 'SSD Used'); my $CMD_LIVE = qq[2>&1 ${SUDO} disi -I diskpool list -v]; my $CMD_LOGS = qq[2>&1 cat local/isi_diskpool]; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze my $header = shift(@$gather); if (!$header) { return } my $header_length = length($header); my $members_length = index($header, 'VHS') - index($header, 'Members') - 1; my $template = generateUnpackTemplate($header, @COLS); my %Pools = (); my $id = 0; foreach my $line (@$gather) { if ($line =~ /^\s*-*$/) { next } if ($line =~ /Unprovisioned drives:(?:\s+(.*))?$/) { if (defined $1 and $1 !~ /none/i) { $CLUSTER_INFO{'Unprovisioned drives'} = $1; debug("CLUSTER_INFO->Unprovisioned drives=$1"); } last; } $line = sprintf("%-${header_length}s", $line); my %unpacked_line; eval { @unpacked_line{@COLS} = unpack($template, $line) }; if (!%unpacked_line) { next } if ($unpacked_line{'Type'}) { $id++ } foreach (keys %unpacked_line) { if ($_ eq 'Members' and length($unpacked_line{$_}) ne $members_length and substr($unpacked_line{$_}, -1) eq ',') { # PSCALE-48039 $unpacked_line{$_} .= ' '; } $Pools{$id}{$_} .= $unpacked_line{$_}; } } if (%Pools) { debug('Pools='.toJSON(\%Pools)) } my %neighborhoods = (); POOL: foreach my $id (sort keys %Pools) { if (!$Pools{$id}{'Type'} or !$Pools{$id}{'Members'} or $Pools{$id}{'Type'} ne 'D') { next } my @members = split(/:bay\S+\s*/, $Pools{$id}{'Members'}); my @neighbors = (); foreach (@members) { push(@neighbors, expandRange($_)); } @neighbors = sort {$a <=> $b} @neighbors; foreach my $n (keys %neighborhoods) { if ($neighborhoods{$n}[0] eq $neighbors[0]) { next POOL } } @{$neighborhoods{$id}} = @neighbors; } if (%neighborhoods) { %{$CLUSTER_INFO{'Neighborhoods'}} = %neighborhoods; debug('CLUSTER_INFO->Neighborhoods='.toJSON(\%neighborhoods)); } } # End gatherNeighborhoods ### # gatherNFSExport: Gathers NFS export information sub gatherNFSExport { return if wasCalled(); my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi nfs exports list --verbose"); } else { $gather = gatherLocal("2>&1 cat local/isi_nfs_exports"); } my $ID = ""; foreach my $line (@$gather) { $ID = $1 if ($line =~ /^\s*ID: (.*)$/ or $line =~ /^Export (.*):$/); next unless $ID; $ID = 0 if $ID =~ /default/i; if ($line =~ /^\s*Zone: (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Zone'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Zone=$1"); } elsif ($line =~ /^\s*Encoding: (.*)$/ or $line =~ /data encoding is (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Encoding'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Encoding=$1"); } elsif ($line =~ /^\s*Paths: (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Paths'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Paths=$1"); } elsif ($line =~ /^\s*Security Type: (.*)$/) { $CLUSTER_INFO{'NFS Export'}{$ID}{'Security Type'} = $1; debug("CLUSTER_INFO->NFS Export->${ID}->Security Type=$1"); } } } # End gatherNFSExport ### # gatherNTPConfig: Gathers NTP Time Server details from the ntp_config.gc file sub gatherNTPConfig { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 cat /ifs/.ifsvar/modules/ntp/ntp_config.gc /var/ifs/ntp_config.gc ]); } else { $gather = gatherLocal(qq[2>&1 cat local/ifsvar_modules.tar/modules/ntp/ntp_config.gc */gconfig_local.tar/var/ifs/ntp_config.gc]); } my @ntp_servers = (); foreach my $line (@$gather) { next unless ($line =~ /^servers.*?\[\s*(.*?)\s*\]/); my $server_list = $1; $server_list =~ s/\\"//g; my @servers = split(/, /, $server_list); foreach my $server (@servers) { my @results = grep {$_ eq $server} @ntp_servers; next if @results; push(@ntp_servers, $server); } } if (@ntp_servers) { @{$CLUSTER_INFO{'NTP Servers'}} = @ntp_servers; debug("CLUSTER_INFO->NTP Servers=".join(",", @ntp_servers)); } } # End gatherNTPConfig ### # gatherNTPdXML: Gathers NTP Time Server details from the ntpd.xml file sub gatherNTPdXML { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 cat /ifs/.ifsvar/etc/mcp/override/ntpd.xml /etc/mcp/override/ntpd.xml]); } else { $gather = gatherLocal(qq[2>&1 cat local/ifsvar_etc.tar/etc/mcp/override/ntpd.xml */local_configs.tar/etc/mcp/override/ntpd.xml */override.tar/override/ntpd.xml]); } my @ntp_servers = (); foreach my $line (@$gather) { next unless ($line =~ /(.*?)<\/server>$/); my $server = $1; my @results = grep {$_ eq $server} @ntp_servers; next if @results; push(@ntp_servers, $server); } if (@ntp_servers) { @{$CLUSTER_INFO{'NTP Servers'}} = @ntp_servers; debug("CLUSTER_INFO->NTP Servers=".join(",", @ntp_servers)); } } # End gatherNTPdXML ### # gatherRemoteSupport: Gather remotesupport config sub gatherRemoteSupport { if (wasCalled()) { return } if (isCurrentOneFS('<7.1')) { return } if (isCurrentOneFS('>=9.1')) { return } # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi_gconfig -t remote-support]); } elsif (-e 'local/isi_remote_support') { $gather = gatherLocal(q[2>&1 cat local/isi_remote_support]); } elsif (-e 'local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/Gconfig.remotesupport_config.sqlite') { $gather = gatherLocal(q[2>&1 sqlite3 local/ifsvar_modules_tardis.tar/modules/tardis/namespaces/Gconfig.remotesupport_config.sqlite 'select * from kv_table']); } elsif (isCurrentOneFS('<8')) { $gather = gatherLocal(q[2>&1 cat */gconfig_local.tar/var/ifs/remote_support.gc]); } # Store relavant information foreach my $line (@$gather) { my ($key, $value) = parseGconfigItem($line); if (!$key or !$value) { next; } elsif ($key eq 'connectemc.enabled' and $value eq 'true') { $CLUSTER_INFO{'RemoteSupport'}{'Enabled'} = 'Yes'; debug('CLUSTER_INFO->RemoteSupport->Enabled=Yes'); } elsif ($key eq 'connectemc.primary_esrs_gateway') { $CLUSTER_INFO{'RemoteSupport'}{'Primary Gateway'} = $value; debug("CLUSTER_INFO->RemoteSupport->Primary Gateway=$value"); } elsif ($key eq 'connectemc.secondary_esrs_gateway') { $CLUSTER_INFO{'RemoteSupport'}{'Secondary Gateway'} = $value; debug("CLUSTER_INFO->RemoteSupport->Secondary Gateway=$value"); } elsif ($key eq 'connectemc.gateway_access_pools') { my ($key, @values) = parseGconfigList($line); foreach my $pool (@values) { $pool =~ s/:/./ } @{$CLUSTER_INFO{'RemoteSupport'}{'Pools'}} = @values; debug("CLUSTER_INFO->RemoteSupport->Pools=[ @values ]"); } elsif ($key eq 'connectemc.remote_support_subnet') { $CLUSTER_INFO{'RemoteSupport'}{'Subnet'} = $value; debug("CLUSTER_INFO->RemoteSupport->Subnet=$value"); } } } # End gatherRemoteSupport sub gatherServices { return if wasCalled(); my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal(qq[2>&1 ${SUDO} isi services -a]); } else { $gather = gatherLocal(qq[2>&1 cat local/isi_services]); } foreach (@$gather) { next unless /^\s*(\S+)\s.*(Enabled|Disabled)$/; $CLUSTER_INFO{'Services'}{$1} = $2; debug("CLUSTER_INFO->Services->${1}=${2}"); } } sub gatherSMBTime { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster(qq[2>&1 ${SUDO} isi_for_array grep running /etc/mcp/override/smbtime.xml]); } else { $gather = gatherCluster(qq[2>&1 grep -H running */override.tar/override/smbtime.xml]); } # Analyze information for issues. foreach my $lnn (sort {$a <=> $b} keys %NODE_INFO) { next unless $gather->{$lnn}; foreach my $line (@{$gather->{$lnn}}) { last if defined $CLUSTER_INFO{'SMBTime'}; if ($line =~ /(.*?)<\/running>/) { $CLUSTER_INFO{'SMBTime'} = $1; } } } } # End gatherSMBTime sub gatherSnapshotCount { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi snap list | grep '^Total'"); } else { $gather = gatherLocal("2>&1 cat local/snapshot_list | grep '^Total'"); } # Analyze information for issues. foreach my $line (@$gather) { if ($line =~ /Total:\s+(\d+)$/) { $CLUSTER_INFO{'Snapshots'}{'Count'} = $1; debug("CLUSTER_INFO->Snapshots->Count=$1"); } } } # End gatherSnapshotCount sub gatherSnapshotSettings { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi snap settings view"); } else { $gather = gatherLocal("2>&1 cat local/snapshot_settings"); } # Analyze information for issues. foreach my $line (@$gather) { if ($line =~ /Autodelete:\s+(\S+)$/) { $CLUSTER_INFO{'Snapshots'}{'Autodelete'} = $1; debug("CLUSTER_INFO->Snapshots->Autodelete=$1"); } } } # End gatherSnapshotSettings sub gatherSnapshotUsage { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi snap snap list -v --format table --state all | grep '^Total'"); } else { $gather = gatherLocal("2>&1 cat local/snapshot_usage | grep '^Total'"); } # Analyze information for issues. foreach my $line (@$gather) { if ($line =~ /^Total:\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { $CLUSTER_INFO{'Snapshots'}{'Count'} = $1; debug("CLUSTER_INFO->Snapshots->Count=$1"); $CLUSTER_INFO{'Snapshots'}{'Size'} = $2; debug("CLUSTER_INFO->Snapshots->Size=$2"); $CLUSTER_INFO{'Snapshots'}{'Reserve'} = $3; debug("CLUSTER_INFO->Snapshots->Reserve=$3"); $CLUSTER_INFO{'Snapshots'}{'Filesystem'} = $4; debug("CLUSTER_INFO->Snapshots->Filesystem=$4"); } } } # End gatherSnapshotUsage ### # gatherSRS: Gathers SRS details sub gatherSRS { if (wasCalled()) { return } if (isCurrentOneFS('<8.1')) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_gconfig -t esrs|grep -v 'api_key']; my $CMD_LOGS = q[2>&1 cat local/isi_esrs|grep -v '^[#+-]']; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal($CMD_LIVE); } else { $gather = gatherLocal($CMD_LOGS); } # Analyze my %gateway_access_pools = (); foreach my $line (@$gather) { my ($key, $value) = parseGconfigItem($line); if (!$key and !$value) { next; } elsif ($key eq 'esrs.enabled' and $value eq 'true') { $CLUSTER_INFO{'SRS'}{'Enabled'} = 'Yes'; debug('CLUSTER_INFO->SRS->Enabled=Yes'); } elsif ($key eq 'esrs.primary_esrs_gateway') { $CLUSTER_INFO{'SRS'}{'Primary Gateway'} = $value; debug("CLUSTER_INFO->SRS->Primary Gateway=$value"); } elsif ($key eq 'esrs.secondary_esrs_gateway') { $CLUSTER_INFO{'SRS'}{'Secondary Gateway'} = $value; debug("CLUSTER_INFO->SRS->Secondary Gateway=$value"); } elsif ($key =~ /^esrs\.gateway_access_pools\.(\d+)\.subnet/) { $gateway_access_pools{$1}{'Subnet'} = $value; } elsif ($key =~ /^esrs\.gateway_access_pools\.(\d+)\.pool/) { $gateway_access_pools{$1}{'Pool'} = $value; } elsif ($key eq 'esrs.eligible_lnns') { (undef, @{$CLUSTER_INFO{'SRS'}{'Nodes'}}) = parseGconfigList($line); debug('CLUSTER_INFO->SRS->Nodes='. "[ @{$CLUSTER_INFO{'SRS'}{'Nodes'}} ]"); } } foreach (keys %gateway_access_pools) { if (!$gateway_access_pools{$_}{'Subnet'} or !$gateway_access_pools{$_}{'Pool'}) { next; } my $pool = sprintf('%s.%s', $gateway_access_pools{$_}{'Subnet'}, $gateway_access_pools{$_}{'Pool'}); push(@{$CLUSTER_INFO{'SRS'}{'Pools'}}, $pool); } if ($CLUSTER_INFO{'SRS'}{'Pools'}) { debug('CLUSTER_INFO->SRS->Pools='. "[ @{$CLUSTER_INFO{'SRS'}{'Pools'}} ]"); } } # End gatherSRS ### # gatherStat: Gathers node information from isi stat -n/isi stat -a output. sub gatherStat { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { if (isCurrentOneFS('>=8')) { $gather = gatherLocal("2>&1 ${SUDO} isi stat -a | grep -E '^(Member|Node|Available|Used)'"); } else { $gather = gatherLocal("2>&1 ${SUDO} isi stat -n | grep -E '^(Member|Node|Available|Used)'"); } } else { $gather = gatherLocal("2>&1 grep -E -h '^(Member|Node|Available|Used)' */isi_stat"); } # Analyze information for issues. my $lnn = 0; my $first_lnn = 0; foreach my $line (@$gather) { next unless $line =~ /:/; my ($name, $value) = split(/:\s*/, $line, 2); if ($name eq "Node LNN") { next unless $value =~ /^\d+$/; $lnn = $value; last if $first_lnn == $lnn; $first_lnn = $lnn unless $first_lnn; } next unless $lnn; if ($name eq "Node ID") { $NODE_INFO{$lnn}{'ID'} = $value; debug("NODE_INFO->${lnn}->ID=$value"); } elsif ($name eq "Node Name") { $NODE_INFO{$lnn}{'Name'} = $value; debug("NODE_INFO->${lnn}->Name=$value"); } elsif ($name eq "Node Health") { $NODE_INFO{$lnn}{'Health'} = $value; debug("NODE_INFO->${lnn}->Health=$value"); } elsif ($name eq "Node SN") { $NODE_INFO{$lnn}{'Serial'} = $value; debug("NODE_INFO->${lnn}->Serial=$value"); } elsif ($name eq "Member of (Node|Disk) (Pool|Pools|Group)") { $NODE_INFO{$lnn}{'Pool'} = $value; debug("NODE_INFO->${lnn}->Pool=$value"); } elsif ($name eq "Node Capacity") { $NODE_INFO{$lnn}{'Capacity'} = $value; debug("NODE_INFO->${lnn}->Capacity=$value"); } elsif ($name eq "Available") { $NODE_INFO{$lnn}{'Available'} = $value; debug("NODE_INFO->${lnn}->Available=$value"); if ($value =~ /\D([0-9\.]+)\%/) { $NODE_INFO{$lnn}{'Percent Available'} = $1; debug("NODE_INFO->${lnn}->Percent Available=$1"); } } elsif ($name eq "Used") { $NODE_INFO{$lnn}{'Used'} = $value; debug("NODE_INFO->${lnn}->Used=$value"); if ($value =~ /\D([0-9\.]+)\%/) { $NODE_INFO{$lnn}{'Percent Used'} = $1; debug("NODE_INFO->${lnn}->Percent Used=$1"); } } } } # End gatherStat ### # gatherStatus: Gathers cluster information from isi status sub gatherStatus { return if wasCalled(); # Gather information from cluster or from logset my $gather = (); if ($RPS{'Live'}) { $gather = gatherLocal("2>&1 ${SUDO} isi status"); } else { $gather = gatherLocal("2>&1 cat */isi_status"); } # Analyze information for issues. my $seen_cluster_name = 0; foreach my $line (@$gather) { my ($hdd, $ssd, $quantifier, $numeric) = ""; if ($line =~ /^Cluster Name:\s+(\S+)$/) { last if $seen_cluster_name; $seen_cluster_name = 1; $CLUSTER_INFO{'Name'} = $1; debug("CLUSTER_INFO->Name=$CLUSTER_INFO{'Name'}"); } elsif ($line =~ /^Cluster Health:\s+\[\s*(\S+)\s*\]$/) { $CLUSTER_INFO{'Health'} = $1; debug("CLUSTER_INFO->Health=$CLUSTER_INFO{'Health'}"); } elsif ($line =~ /^VHS Size:\s+(\S+)$/) { $CLUSTER_INFO{'VHS Size'} = $1; debug("CLUSTER_INFO->VHS Size=$CLUSTER_INFO{'VHS Size'}"); } elsif ($line =~ /^Size:\s+(\S+\s+\(.*\))\s+(\S+\s+\(.*\))$/) { $CLUSTER_INFO{'HDD Storage'} = $1; debug("CLUSTER_INFO->HDD Storage=$1"); $CLUSTER_INFO{'SSD Storage'} = $2; debug("CLUSTER_INFO->SSD Storage=$2"); if ($CLUSTER_INFO{'HDD Storage'} =~ /\((\S+)\s/) { $CLUSTER_INFO{'HDD Size'} = convertToNumeric($1); debug("CLUSTER_INFO->HDD Size=$CLUSTER_INFO{'HDD Size'}"); } if ($CLUSTER_INFO{'SSD Storage'} =~ /\((\S+)\s/) { $CLUSTER_INFO{'SSD Size'} = convertToNumeric($1); debug("CLUSTER_INFO->SSD Size=$CLUSTER_INFO{'SSD Size'}"); } } elsif ($line =~ /^Used:\s+(\S+\s+\(.*\))\s+(\S+\s+\(.*\))$/) { $CLUSTER_INFO{'HDD Used'} = $1; debug("CLUSTER_INFO->HDD Used=$1"); $CLUSTER_INFO{'SSD Used'} = $2; debug("CLUSTER_INFO->SSD Used=$2"); if ($CLUSTER_INFO{'HDD Used'} =~ /\((\d+)%/) { $CLUSTER_INFO{'HDD Percent Used'} = $1; debug("CLUSTER_INFO->HDD Percent Used=$CLUSTER_INFO{'HDD Percent Used'}"); } if ($CLUSTER_INFO{'SSD Used'} =~ /\((\d+)%/) { $CLUSTER_INFO{'SSD Percent Used'} = $1; debug("CLUSTER_INFO->SSD Percent Used=$CLUSTER_INFO{'SSD Percent Used'}"); } } elsif ($line =~ /^\s*(\d+)\|/) { my $lnn = $1; my @columns = split(/\s*\|\s*/, $line); foreach (@columns) { if (/([\d\.]+)%/) { $NODE_INFO{$lnn}{'Percent Used'} = $1; debug("NODE_INFO->${lnn}->Percent Used=$1"); last; } } $NODE_INFO{$lnn}{'SSD Storage'} = $columns[$#columns]; debug("NODE_INFO->${lnn}->SSD Storage=$NODE_INFO{$lnn}{'SSD Storage'}"); } } } # End gatherStatus ### # gatherUpgradeCluster: Gathers information from isi upgrade cluster view sub gatherUpgradeCluster { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi upgrade cluster view]; my $CMD_LOGS = q[2>&1 cat */isi_upgrade_cluster_view local/isi_upgrade_view]; # Gather my $gather = gatherLocal(($RPS{'Live'}?$CMD_LIVE:$CMD_LOGS)); # Save foreach (@$gather) { if (/Cluster Upgrade State: (.*)/) { if ($CLUSTER_INFO{'Cluster Upgrade State'}) { next } $CLUSTER_INFO{'Cluster Upgrade State'} = $1; debug("CLUSTER_INFO->Cluster Upgrade State=$1"); } if (/Current Upgrade Activity: (.*)/) { if ($CLUSTER_INFO{'Current Upgrade Activity'}) { next } $CLUSTER_INFO{'Current Upgrade Activity'} = $1; debug("CLUSTER_INFO->Current Upgrade Activity=$1"); } } } # End gatherUpgradeCluster ### # gatherUptime: Gathers and calculates each nodes uptime sub gatherUptime { if (wasCalled()) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array uptime]; my $CMD_LOGS = q[2>&1 grep -H . */uptime]; # adjustment: the difference between the log gather time and script run time my $adjustment = 0; $adjustment = $RPS{'Runtime'} - $RPS{'Timestamp'}; debug("adjustment=$adjustment seconds"); # Gather my $gather = gatherCluster(($RPS{'Live'}?$CMD_LIVE:$CMD_LOGS)); # Anaylze foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { missing($lnn); next; } foreach my $line (@{$gather->{$lnn}}) { $NODE_INFO{$lnn}{'uptime_gathered'} = $line; debug("NODE_INFO->${lnn}->uptime_gathered=$line"); # Parse current time if ($line =~ /(\d+:\d+[AP]M)/) { $NODE_INFO{$lnn}{'time'} = $1; debug("NODE_INFO->${lnn}->time->$NODE_INFO{$lnn}{'time'}"); } # uptime_seconds: the number of seconds a node has been up for my $uptime_seconds = 0; if ($line =~ /\s(\d+)\s+day/) { $uptime_seconds += $1 * 86400; } if ($line =~ /\s(\d+)\s+hr/) { $uptime_seconds += $1 * 3600; } if ($line =~ /\s(\d+)\s+min/) { $uptime_seconds += $1 * 60; } if ($line =~ /\s(\d+)\s+sec/) { $uptime_seconds += $1; } if ($line =~ /\s(\d+):(\d+),/) { $uptime_seconds += $1 * 3600 + $2 * 60; } # If uptime_seconds is 0, there was no uptime information found if (!$uptime_seconds) { next } # Total uptime is the number of seconds of reported uptime plus the # adjustment variable if running on a logset $NODE_INFO{$lnn}{'uptime'} = $uptime_seconds + $adjustment; debug("NODE_INFO->${lnn}->uptime->$NODE_INFO{$lnn}{'uptime'}"); } } } # End gatherUptime $CHECKS{'checkJobHistory'}{'Description'} = 'Checks job history for issues, currently just MediaScan issues.'; sub checkJobHistory { if (wasCalled()) { return } printCheck('Jobs History'); my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array]. q[ 'grep -hE "MediaScan.*Succeeded$" /var/log/messages']; my $CMD_LOGS = q[2>&1 grep -HE 'MediaScan.*Succeeded$']. q[ */varlog.tar/log/messages]; my $MSG_NO = 'No MediaScan job history found'; my $MSG_90 = 'Media scan not completed in last 90 days'; my $MSG_60 = 'Media scan not completed in last 60 days'; # Gather my $gather = gatherCluster(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze my $mr_mediascan = 0; foreach my $lnn (@LNNS) { if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/\s*(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/) { my $timestamp = timelocal($6, $5, $4, $3, $2 - 1, $1); debug($timestamp); if ($timestamp > $mr_mediascan) { $mr_mediascan = $timestamp } } } } # Flag if (!$mr_mediascan) { warning($MSG_NO); } elsif ($RPS{'Timestamp'} - $mr_mediascan > 90 * 24 * 60 * 60) { warning($MSG_90); } elsif ($RPS{'Timestamp'} - $mr_mediascan > 60 * 24 * 60 * 60) { info($MSG_60); } printResults(); } # End checkJobHistory ### # checkJobs: Checks jobs $CHECKS{'checkJobs'}{'Description'} = 'Checks jobs'; sub checkJobs { if (wasCalled()) { return } printCheck('Jobs'); gatherJobStatus(); gatherJobTypes(); gatherJobHistory(); gatherETA518924(); # Setup my @job_types = sort {$a cmp $b} keys %{$CLUSTER_INFO{'Jobs'}}; my $max_type = 10; my $max_policy = 6; foreach my $type (keys %{$CLUSTER_INFO{'Jobs'}}) { my $length_type = length($type); if ($length_type > $max_type) { $max_type = $length_type } if (!defined $CLUSTER_INFO{'Jobs'}{$type}{'Policy'}) { next } my $length_policy = length($CLUSTER_INFO{'Jobs'}{$type}{'Policy'}); if ($length_policy > $max_policy) { $max_policy = $length_policy } } my $SPF_JOB = "%-${max_type}s %-8s %-3s %-${max_policy}s %-24s %s"; # Output job details if (!$RPS{'JSON'}) { details(sprintf($SPF_JOB, 'Type', 'Enabled', 'Pri', 'Policy', 'Most Recent Completion', 'Schedule')); foreach my $type (@job_types) { my %job = %{$CLUSTER_INFO{'Jobs'}{$type}}; details(sprintf($SPF_JOB, $type, $job{'Enabled'} || '-', $job{'Priority'} || '-', $job{'Policy'} || '-', $job{'Last'}?scalar(localtime($job{'Last'})):'-', $job{'Schedule'} || '-')); } } # Flag if ($RPS{'ETA'}{'518924'}{'Severity'}) { if (isJobEnabled('Collect') or isJobEnabled('MultiScan')) { # Flag to disable the Collect/MultiScan jobs per ETA 518924 } } else { if (isJobDisabled('Collect') or isJobDisabled('MultiScan')) { # Flag to enable the Collect/MultiScan jobs } } printResults(); } # End checkJobs ### # checkJobStatus: Checks for running jobs that would impact an upgrade. $CHECKS{'checkJobStatus'}{'Description'} = 'Checks for running jobs that would impact an upgrade'; sub checkJobStatus { if (wasCalled()) { return } printCheck('Jobs Status'); gatherJobStatus(); # Setup my $SPF_ONEFS = '%s is currently %s and must complete prior to any OneFS upgrades'; my $SPF_UPG = '%s is currently %s and must complete prior to any upgrade activities'; my $SPF_CANCEL = '%s is currently %s and must be cancelled prior to any OneFS upgrades'; my $SPF_MSNOT = 'MediaScan fixes available but not in %s'; my $MSG_MS = 'MediaScan fixes available with OneFS upgrade'; my $MSG_UPGRADE_DISABLED = 'The Upgrade job is currently disabled and should be enabled by'. ' running the following command:'. ' isi job type modify upgrade --enabled=true'; my $MSG_KB = 'For more information see KB 000039736'. ' [https://www.dell.com/support/kbdoc/en-us/000039736/'. 'isilon-on-cluster-analysis-jobs-status]'; # Analyze foreach my $type (keys %{$CLUSTER_INFO{'Jobs'}}) { if (!$CLUSTER_INFO{'Jobs'}{$type}{'State'}) { next } my $state = $CLUSTER_INFO{'Jobs'}{$type}{'State'}; if ($type eq 'Upgrade') { if (isDestinationOneFS()) { fail(sprintf($SPF_ONEFS, $type, $state)); } else { warning(sprintf($SPF_ONEFS, $type, $state)); } } elsif ($type =~ /IntegrityScan|FlexProtect/i) { fail(sprintf($SPF_UPG, $type, $state)); } elsif ($type =~ /dedupe/i and isDestinationOneFS('>=8')) { critical(sprintf($SPF_CANCEL, $type, $state)); } } if (isDestinationOneFS('<7.1.1.5') or isDestinationOneFS(qr/^7\.2\.0\.[012]$/)) { warning(sprintf($SPF_MSNOT, isDestinationOneFS())); } if (!isDestinationOneFS() and (isCurrentOneFS('<7.1.1.5') or isCurrentOneFS(qr/^7\.2\.0\.[012]$/))) { warning($MSG_MS); } if (isJobDisabled('Upgrade')) { fail($MSG_UPGRADE_DISABLED) } if (isResults(&WARN)) { info($MSG_KB) } printResults(); } # End checkJobStatus $CHECKS{'checkKB537785'}{'Description'} = 'Check for criteria of KB 537785'; sub checkKB537785 { if (wasCalled()) { return } if (!(isCurrentOneFS(qr/^8\.2\.[01]\./) or isDestinationOneFS(qr/^8\.2\.[01]\./))) { return unsupported('Supported in OneFS 8.2.0 and OneFS 8.2.1'); } if (!$RPS{'Live'} and !isDestinationOneFS(qr/^8\.2\.[01]\./)) { return } # Setup printCheck('KB 537785'); my $SYSCTL = 'efs.pdm.check_valid_tag_op'; my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array 'sysctl $SYSCTL']; my $MSG_SYSCTL = 'The sysctl setting efs.pdm.check_valid_tag_op should be'. ' changed to 0 by running the following command: isi_sysctl_cluster'. ' efs.pdm.check_valid_tag_op=0'; my $SPF_UPGRADE = 'Immediately following the upgrade to %s, run the'. ' following command to change the sysctl setting'. ' efs.pdm.check_valid_tag_op to 0: isi_sysctl_cluster'. ' efs.pdm.check_valid_tag_op=0'; my $MSG_KB = 'For more details see KB 537785'. ' [https://www.dell.com/support/kbdoc/537785 - The DomainTag job may'. ' result in a multiple nodes panic]'; # Gather my $gather = (); if ($RPS{'Live'}) { $gather = gatherCluster($CMD_LIVE); } # Analyze my $enabled = 0; foreach my $lnn (@LNNS) { if (!$gather or $enabled) { last } if (!$gather->{$lnn}) { next } foreach (@{$gather->{$lnn}}) { if (/^\s*${SYSCTL}[:=]\s*(\d)/) { $enabled = $1 } } } # Flag if ($enabled) { if ($RPS{'Target'}) { critical($MSG_SYSCTL); } else { fail($MSG_SYSCTL); } } elsif (isDestinationOneFS(qr/^8\.2\.[01]\./) and isCurrentOneFS('<8.2')) { critical(sprintf($SPF_UPGRADE, $RPS{'Target'})); } # Output if (isResults(&WARN)) { info($MSG_KB) } printResults(); } # gatherJobHistory: Gathers job history from /var/log/messages files sub gatherJobHistory { if (wasCalled()) { return } import Parsers qw( parseTime ); my $CMD_LIVE = qq[2>&1 ${SUDO} isi_for_array -Q]. q[ 'zgrep -a -h Succeeded$ /var/log/messages*']. q( | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}.*isi_job_d.*Succeeded$'). q( | awk '{print $1" "$(NF-1)}'). q( | sed -e 's/\[.*\]$//'). q[ | sort -r -k2 | uniq -f1]; my $CMD_LOGS = q[2>&1 zgrep -a -h Succeeded$ */varlog.tar/log/messages*]. q( | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}.*isi_job_d.*Succeeded$'). q( | awk '{print $1" "$(NF-1)}'). q( | sed -e 's/\[.*\]$//'). q[ | sort -r -k2 | uniq -f1]; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Analyze foreach (@$gather) { my ($time, $type) = split(/ /); my $timestamp = parseTime($time); $CLUSTER_INFO{'Jobs'}{$type}{'Last'} = $timestamp; debug("CLUSTER_INFO->Jobs->${type}->Last=$timestamp"); } } # End gatherJobHistory ### # gatherJobStatus7: Gathers job information in OneFS 7 and earlier sub gatherJobStatus7 { if (wasCalled()) { return } if (isCurrentOneFS('>=8')) { debug('Supported in OneFS 7 and earlier'); return; } # Setup my $CMD_LIVE_7 = qq[2>&1 ${SUDO} isi_classic job status -v]; my $CMD_LIVE_6 = qq[2>&1 ${SUDO} isi job status -v]; my $CMD_LOGS = q[2>&1 cat local/isi_job_status]; # Gather my $gather = []; if (isLive()) { if (isCurrentOneFS('>=7')) { $gather = gatherLocal($CMD_LIVE_7); } else { $gather = gatherLocal($CMD_LIVE_6); } } else { $gather = gatherLocal($CMD_LOGS) } # Save my $state = ''; foreach (@$gather) { if (/^Running jobs:/) { $state = 'Running' } if (/^Paused and waiting jobs:/) { $state = 'Paused' } if (/^No (running|paused or waiting|failed) jobs/) { $state = '' } if (/^Failed jobs:|^Recent job results/) { last } if (/^(\w+)\[\d+\]/) { debug("CLUSTER_INFO->Jobs->${1}->State=$state"); $CLUSTER_INFO{'Jobs'}{$1}{'State'} = $state; } } } # End gatherJobStatus7 ### # gatherJobStatus8: Gathers current jobs in OneFS 8 and later sub gatherJobStatus8 { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { debug('Supported in OneFS 8 and later'); return; } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi job list]; my $CMD_LOGS = q[2>&1 cat local/isi_job_list]; my @COLS = ('ID', 'Type', 'State', 'Impact','Running'); # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Save my $unpack_template = ''; foreach (@$gather) { if (/^ID/) { $unpack_template = generateUnpackTemplate($_, @COLS) } if (!$unpack_template or /^-*$/ or !/^\d/) { next } eval { my ($id, $type, $state, undef, $running) = unpack($unpack_template, $_); debug("CLUSTER_INFO->Jobs->${type}->State=$state"); $CLUSTER_INFO{'Jobs'}{$type}{'State'} = $state; debug("CLUSTER_INFO->Jobs->${type}->Running=$running"); $CLUSTER_INFO{'Jobs'}{$type}{'Running'} = $running; }; } } # End gatherJobStatus8 ### # gatherJobStatus: Gathers job information. sub gatherJobStatus { return if wasCalled(); gatherJobStatus7(); gatherJobStatus8(); } # End gatherJobStatus sub gatherJobTypes { if (wasCalled()) { return } if (isCurrentOneFS('<8')) { return } # Setup my $CMD_LIVE = qq[2>&1 ${SUDO} isi job types list --format list -v --all]; my $CMD_LOGS = qq[2>&1 cat local/isi_job_schedule]; my @FIELDS = ('Description','Enabled','Policy','Schedule','Priority'); my $RE_CAPTURE = '^\s*('.join('|', @FIELDS).'):\s*(\S.*)$'; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Save my $type = ''; foreach (@$gather) { if (/^[\s-]*$/) { } elsif (/^\s*ID:\s*(\S+)$/) { $type = $1; } elsif (!$type) { } elsif (/$RE_CAPTURE/) { $CLUSTER_INFO{'Jobs'}{$type}{$1} = $2; debug("CLUSTER_INFO->Jobs->${type}->$1=$2"); } } } # End gatherJobTypes sub isJobDisabled { my ($type) = @_; if (!$type) { return 0 } gatherJobTypes(); if (!defined $CLUSTER_INFO{'Jobs'}{$type}) { return 0 } if (!defined $CLUSTER_INFO{'Jobs'}{$type}{'Enabled'}) { return 0 } if ($CLUSTER_INFO{'Jobs'}{$type}{'Enabled'} eq 'No') { return 1 } } sub isJobEnabled { my ($type) = @_; if (!$type) { return 0 } gatherJobTypes(); if (!defined $CLUSTER_INFO{'Jobs'}{$type}) { return 0 } if (!defined $CLUSTER_INFO{'Jobs'}{$type}{'Enabled'}) { return 0 } if ($CLUSTER_INFO{'Jobs'}{$type}{'Enabled'} eq 'Yes') { return 1 } } ### # checkKB495911: Checks for an issue where having a large number of # SyncIQ report files causes a tar process to delay other upgrade # processes potentially leaving the cluster in a temporary DU scenario # for an extended period of time. sub checkKB495911 { if (wasCalled()) { return } gatherSyncIQ(); if (!$CLUSTER_INFO{'SyncIQ'}{'Source'}) { return } # Setup my $SQL_QUERY = q['select policy_id, count(policy_id) from reports group by policy_id']; my $FILE_LIVE = '/ifs/.ifsvar/modules/tsm/sched/reports/reports.db'; my $FILE_LOGS = 'local/ifsvar_modules.tar/modules/tsm/sched/reports/reports.db'; my $CMD_LIVE = qq[2>&1 sqlite3 $FILE_LIVE $SQL_QUERY]; my $CMD_LOGS = qq[2>&1 sqlite3 $FILE_LOGS $SQL_QUERY]; my $SPF_VERBOSE = '%-32s %-20s %-20s %s'; my $SPF_EST = 'The estimated total number of SyncIQ report files is %s, this may extend the duration of cluster unavailability during an upgrade to OneFS %s. A fix for this is available in OneFS 8.0.0.6, OneFS 8.0.1.2, and OneFS 8.1.0.1 and later OneFS versions.'; my $MSG_DBL = 'The following policies have estimated more than double the amount of SyncIQ reports they should have'; my $SPF_POL = '%-30s %-20s %s'; my $MAX_FILES_COUNT = 100000; # Gather my $gather = (); if (isLive() and -e $FILE_LIVE) { $gather = gatherLocal($CMD_LIVE); } elsif (!isLive() and -e $FILE_LOGS) { $gather = gatherLocal($CMD_LOGS); } # Analyze my $total_estimated_files_count = 0; my %policies_with_report_issues = (); verboseHeader('SyncIQ Reports'); verbose(sprintf($SPF_VERBOSE, 'Policy ID', 'Policy Name', 'Report Max Count', 'Estimated Count')); foreach my $line (@$gather) { if ($line !~ /\|/) { next } my ($pid, $estimate) = split(/\|/, $line); $total_estimated_files_count += $estimate; if (!$CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}) { next } my $policy = $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}; if (!$policy->{'Report Max Count'} or !$policy->{'Name'}) { next } verbose(sprintf($SPF_VERBOSE, $pid, $policy->{'Name'}, $policy->{'Report Max Count'}, $estimate)); if ($estimate < 2000 or $estimate < $policy->{'Report Max Count'} * 2) { next } $policies_with_report_issues{$pid} = $estimate; } verbose("Total estimated files count: $total_estimated_files_count"); # Flag if (isCurrentOneFS('<8') and isDestinationOneFS(qr/^8\.0\.0\.[012345]$|^8\.0\.1\.[01]$|^8\.1\.0\.0$/) and $total_estimated_files_count > $MAX_FILES_COUNT) { addCustomerInformation(critical(sprintf($SPF_EST, $total_estimated_files_count, isDestinationOneFS()))); } if (%policies_with_report_issues) { warning($MSG_DBL); warning(sprintf($SPF_POL, 'Policy Name', 'Report Max Count', 'Estimated Count')); foreach my $pid (keys %policies_with_report_issues) { my $policy = $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}; warning(sprintf($SPF_POL, $policy->{'Name'}, $policy->{'Report Max Count'}, $policies_with_report_issues{$pid})); } } } # End checkKB495911 $CHECKS{'checkSyncIQ'}{'Description'} = 'Gathers source and target SyncIQ information and reports partners of SyncIQ. Checks for an issue where having a large number of SyncIQ report files causes a tar process to delay other upgrade processes potentially leaving the cluster in a temporary DU scenario for an extended period of time.'; sub checkSyncIQ { if (wasCalled()) { return } checkKB521778(); printCheck('SyncIQ'); gatherSyncIQ(); gatherFlexNet(); checkKB495911(); my $SPF_SYNCIQ = '%-32s %-30s %-10s %-20s %s'; my $MSG_KB480064 = 'There is a known issue with SyncIQ and Dynamic pools in OneFS versions 7.2.1.0 - 7.2.1.2, for more details see KB480064 [https://www.dell.com/support/kbdoc/480064 - SyncIQ error connecting to a local resource. Error retrieving IP addresses for primary cluster: Local cluster has no external IP addresses]'; my $MSG_DYNAMIC = 'SyncIQ best practice is to use static network pools, and all network pools are dynamic.'; my $SPF_RESTRICTED = 'SyncIQ best practice is to use static network pools, and the following SyncIQ policies are restricted to use a dynamic network pool: %s'; my $MSG_COMPLIANCE = 'As this cluster is in Compliance mode and is upgrading from a version prior to OneFS 8.2.x, both the SyncIQ source and SyncIQ target clusters must be upgraded to the same versions of OneFS and patches before restarting SyncIQ backups. If the SyncIQ source and SyncIQ target clusters are not on the same versions, SyncIQ backups will fail. This issue is resolved in OneFS 8.2.x'; my $MSG_PAUSE = 'SyncIQ policies and jobs must be cancelled or paused before upgrading the OneFS operating system.'; my $MSG_KB489059 = 'For more details see KB489059 [https://www.dell.com/support/kbdoc/489059 - Isilon On-Cluster Analysis - SyncIQ]'; # Analyze incoming SyncIQ my @source_clusters = (); verboseHeader('Incoming SyncIQ'); if ($CLUSTER_INFO{'SyncIQ'}{'Target'}) { verbose(sprintf($SPF_SYNCIQ, 'Policy ID', 'Policy Name', 'Timestamp', 'Status', 'Source Cluster')); foreach my $pid (keys %{$CLUSTER_INFO{'SyncIQ'}{'Target'}}) { my %policy = %{$CLUSTER_INFO{'SyncIQ'}{'Target'}{$pid}}; # Display verbose(sprintf($SPF_SYNCIQ, $pid, $policy{'Name'}, $policy{'Timestamp'}, $policy{'Last Job State'}, $policy{'Source'})); # Add as a source cluster if it is not already one my @results = grep(/^$policy{'Source'}$/i, @source_clusters); if (!@results) { push(@source_clusters, $policy{'Source'}) } } } if (@source_clusters) { details('Target for: '.join(', ', @source_clusters)) } # Gather a list of SyncIQ clusters that are a target from this cluster my @target_clusters = (); # And store the policy name if the policy last job state was not successful my @failed_policies = (); # And store the policy name if the policy restricts use to a Dynamic network pool my @restricted_to_dynamic_pool = (); verboseHeader('Outgoing SyncIQ'); verbose(sprintf($SPF_SYNCIQ, 'Policy ID', 'Timestamp', 'Policy Name', 'Destination Cluster')); if ($CLUSTER_INFO{'SyncIQ'}{'Source'}) { foreach my $pid (keys %{$CLUSTER_INFO{'SyncIQ'}{'Source'}}) { my %siq_outgoing = %{$CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}}; # Next if there is no policy name if (!$siq_outgoing{'Name'} or !$siq_outgoing{'Timestamp'} or !$siq_outgoing{'Target'}) { next } verbose(sprintf($SPF_SYNCIQ, $pid, $siq_outgoing{'Timestamp'}, $siq_outgoing{'Name'}, $siq_outgoing{'Target'})); # Next if the most recent start was over 90 days ago if ($RPS{'Timestamp'} - $siq_outgoing{'Timestamp'} >= 7776000) { next } # Gather a list of policies that are not finished/scheduled/running/- if ($siq_outgoing{'Enabled'} and $siq_outgoing{'Last Job State'} and $siq_outgoing{'Last Job State'} !~ /^(-|finished|scheduled|running)/i) { push(@failed_policies, $siq_outgoing{'Name'}); } # Gather a list of policies that are restricted to use Dynamic IP pools if ($siq_outgoing{'Enabled'} and $siq_outgoing{'Restrict Source Nodes'} and $CLUSTER_INFO{'FlexNet'}{'Pools'}{$siq_outgoing{'Restrict Source Nodes'}}{'Allocation Method'} and $CLUSTER_INFO{'FlexNet'}{'Pools'}{$siq_outgoing{'Restrict Source Nodes'}}{'Allocation Method'} =~ /dynamic/i) { push(@restricted_to_dynamic_pool, $siq_outgoing{'Name'}); } # Compile a list of target clusters my @results = grep(/$siq_outgoing{'Target'}/i, @target_clusters); if (@results) { next } push(@target_clusters, $siq_outgoing{'Target'}) } } if (@target_clusters) { details('Source to: '.join(', ', @target_clusters)) } # Flag for this cluster being part of SyncIQ, used in the upgrade plan if (@source_clusters or @target_clusters) { $RPS{'SyncIQ'} = 'Yes' } # Check if there are only Dynamic pools configured when it is determined there is SyncIQ in use my $only_dynamic_pools_available = 0; if (@target_clusters or @source_clusters) { $only_dynamic_pools_available = 1; foreach (keys %{$CLUSTER_INFO{'FlexNet'}{'Pools'}}) { if (!$CLUSTER_INFO{'FlexNet'}{'Pools'}{$_}{'Allocation Method'}) { next } if ($CLUSTER_INFO{'FlexNet'}{'Pools'}{$_}{'Allocation Method'} =~ /static/i) { $only_dynamic_pools_available = 0; } } } if ($only_dynamic_pools_available or @restricted_to_dynamic_pool) { # Flag a known issue with 7.2.1.0 - 7.2.1.2. if ((!$RPS{'Target'} and isCurrentOneFS(qr/^7\.2\.1\.[012]$/)) or ($RPS{'Target'} and $RPS{'Target'} =~ /^7\.2\.1\.[012]$/)) { fail($MSG_KB480064); } # Warn about not using static pools for replication if ($only_dynamic_pools_available) { addCustomerInformation(warning($MSG_DYNAMIC)); } else { addCustomerInformation(warning(sprintf($SPF_RESTRICTED, join(', ', @restricted_to_dynamic_pool)))); } } if ($RPS{'SyncIQ'} and isCompliance() and isCurrentOneFS('<8.2') and (($RPS{'Target'} and compareVersions($RPS{'Target'}, '8.0.1') >= 0) or ($RPS{'Pre Upgrade Patches'} and isCurrentOneFS(qr/^8\.1/)))) { addCustomerInformation(warning($MSG_COMPLIANCE)); } if ($RPS{'SyncIQ'} and isOneFSUpgrade()) { addCustomerInformation(warning($MSG_PAUSE)); } if (isResults(&WARN)) { info($MSG_KB489059) } printResults(); } # End checkSyncIQ # gatherSyncIQPoliciesGC: Gather SyncIQ policy information from GConfig sub gatherSyncIQPoliciesGC { if (wasCalled()) { return } # Setup my %SECTIONS = ( 'common.name' => 'Name', 'common.pid' => 'ID', 'common.state' => 'Enabled', 'target.dst_cluster_name' => 'Target', 'target.restrict_by' => 'Restrict Source Nodes', 'scheduler.max_reports' => 'Report Max Count', 'scheduler.rotate_report_period' => 'Report Max Age', ); my @SECTIONS = keys %SECTIONS; my $RE_ID = qr/^policy\.(\d+)/; local $" = '|'; my $CMD_GREP = qq[egrep -h '(^policy\.[^.]+ |@SECTIONS)']; my $CMD_LIVE = qq[2>&1 $CMD_GREP /ifs/.ifsvar/modules/tsm/config/siq-policies.gc]; my $CMD_LOGS = qq[2>&1 $CMD_GREP local/ifsvar_modules.tar/modules/tsm/config/siq-policies.gc]; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Parse my %policies = (); my $index = ''; foreach my $line (@$gather) { if ($line =~ $RE_ID) { $index = $1 } if (!defined $index or $line !~ /^policy\.$index\./) { next } my ($key, $value) = parseGconfigItem($line); ($key) = grep {$key =~ /$_/} @SECTIONS; if (!$key) { next } if ($key eq 'common.state' and $value ne 'on') { next } if ($key eq 'target.restrict_by' and $value eq '') { next } $policies{$index}{$SECTIONS{$key}} = $value; } # Store foreach my $index (keys %policies) { my $policy = $policies{$index}; if (!$policy->{'ID'}) { next } my $pid = $policy->{'ID'}; foreach my $key (keys %{$policy}) { $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{$key} = $policy->{$key}; debug("CLUSTER_INFO->SyncIQ->Source->${pid}->${key}=$policy->{$key}"); } } } # End gatherSyncIQPoliciesGC # gatherSyncIQPoliciesList: Check SyncIQ policy information sub gatherSyncIQPoliciesList { if (wasCalled()) { return } # Setup my @SECTIONS = ( 'ID', 'Name', 'Action', 'Enabled', 'Target', 'Source Subnet', 'Source Pool', 'Last Job State', 'Last Started', 'Report Max Count', 'Report Max Age'); local $" = '|'; my $CMD_GREP = qq[egrep '^ *(@SECTIONS):']; my $CMD_LIVE = qq[2>&1 ${SUDO} isi sync pol list -v | $CMD_GREP]; my $CMD_LOGS = qq[2>&1 $CMD_GREP -h */isi_sync_policy]; my $RE_DATETIME = qr/(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)/; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Parse my %siq_policies = (); my $pid = ''; foreach (@$gather) { s/^ *//; my ($key, $value) = split(/:\s+/, $_, 2); if ($value eq '-') { next } if ($key eq 'ID') { $pid = $value; if ($siq_policies{$pid}) { last } } if ($pid and $key and $value) { $siq_policies{$pid}{$key} = $value; } } # Store foreach my $pid (keys %siq_policies) { my $policy = $siq_policies{$pid}; # Calculate timestamp using last started if ($policy->{'Last Started'} and $policy->{'Last Started'} =~ $RE_DATETIME) { $policy->{'Timestamp'} = timelocal($6, $5, $4, $3, $2 - 1, $1); } # Skip policies last started over 90 days if (defined $policy->{'Timestamp'} and $RPS{'Timestamp'} - $policy->{'Timestamp'} > 7776000) { next; } # Delete the Enabled key when the policy is not enabled if ($policy->{'Enabled'} and $policy->{'Enabled'} ne 'Yes') { delete $policy->{'Enabled'}; } # Set Restrict Source Nodes to the Subnet:Pool specified if ($policy->{'Source Pool'} and $policy->{'Source Subnet'}) { $policy->{'Restrict Source Nodes'} = "$policy->{'Source Subnet'}:$policy->{'Source Pool'}"; } # Convert Report Max Age to seconds if ($policy->{'Report Max Age'}) { $policy->{'Report Max Age'} = convertDurationToSeconds($policy->{'Report Max Age'}); } # Store to CLUSTER_INFO foreach my $key (keys %{$policy}) { $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{$key} = $policy->{$key}; debug("CLUSTER_INFO->SyncIQ->Source->${pid}->${key}=$policy->{$key}"); } debug("CLUSTER_INFO->SyncIQ->Source->${pid}=".toJSON($policy)); } } # End gatherSyncIQPoliciesList # gatherSyncIQSourceRecords: Gather outgoing SyncIQ information from record files sub gatherSyncIQSourceRecords { if (wasCalled()) { return } # Setup my @SECTIONS = ('policy-record .*', 'start-sched', 'end-sched'); my @FILES_LIVE = ('/ifs/.ifsvar/modules/tsm/config/source_record.xml', '/ifs/.ifsvar/modules/tsm/config/source_records/*.xml'); my @FILES_LOGS = ('local/ifsvar_modules.tar/modules/tsm/config/source_record.xml', 'local/ifsvar_modules.tar/modules/tsm/config/source_records/*.xml'); local $"='|'; my $CMD_GREP = qq[egrep '<(@SECTIONS)>']; my $RE_PID = qr/policy-id="([^"]*)"/; my $RE_SECTION = qr/<(@SECTIONS)>([^<]*)&1 $CMD_GREP @FILES_LIVE]; my $CMD_LOGS = qq[2>&1 $CMD_GREP @FILES_LOGS]; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Parse my %siq_outgoing = (); my $pid = undef; foreach my $line (@$gather) { if ($line =~ $RE_PID) { $pid = $1 } if (!defined $pid) { next } if ($line =~ $RE_SECTION) { $siq_outgoing{$pid}{$1} = $2 } } # Store foreach my $pid (keys %siq_outgoing) { # Filter out policies with missing details or more than 90 days since last ran if ($siq_outgoing{$pid}{'end-sched'} and $siq_outgoing{$pid}{'start-sched'} and $RPS{'Timestamp'} - $siq_outgoing{$pid}{'end-sched'} <= 7776000) { $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid} = $siq_outgoing{$pid}; debug("CLUSTER_INFO->SyncIQ->Source->${pid}=".toJSON($siq_outgoing{$pid})); } } } # End gatherSyncIQSourceRecords # gatherSyncIQSourceReports: Gather outgoing SyncIQ information from report files sub gatherSyncIQSourceReports { if (wasCalled()) { return } if (!$CLUSTER_INFO{'SyncIQ'}{'Source'}) { return } # Setup my @SECTIONS = ('summary\.total\.state', 'summary\.errors.*msg'); local $"='|'; my $CMD_GREP = qq[egrep '^(@SECTIONS)']; my $SPF_LIVE = qq[2>&1 $CMD_GREP /ifs/.ifsvar/modules/tsm/sched/reports/%s/report-%s.gc]; my $SPF_LOGS = qq[2>&1 $CMD_GREP local/ifsvar_modules.tar/modules/tsm/sched/reports/%s/report-%s.gc]; foreach my $pid (keys %{$CLUSTER_INFO{'SyncIQ'}{'Source'}}) { my %policy = %{$CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}}; if (!$policy{'end-sched'}) { next } my $CMD_LIVE = sprintf($SPF_LIVE, $pid, $policy{'end-sched'}); my $CMD_LOGS = sprintf($SPF_LOGS, $pid, $policy{'end-sched'}); # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Parse foreach my $line (@$gather) { my ($key, $value) = parseGconfigItem($line); $CLUSTER_INFO{'SyncIQ'}{'Source'}{$pid}{$key} = $value; debug("CLUSTER_INFO->SyncIQ->Source->${pid}->${key}=$value"); } } } # End gatherSyncIQSourceReports ### # gatherSyncIQ: Gathers SyncIQ policies and source and target records. sub gatherSyncIQ { if (wasCalled()) { return } gatherSyncIQTargetRecords(); gatherSyncIQSourceRecords(); gatherSyncIQSourceReports(); gatherSyncIQPoliciesGC(); gatherSyncIQPoliciesList(); debug('CLUSTER_INFO->SyncIQ='.toJSON($CLUSTER_INFO{'SyncIQ'})); } # End gatherSyncIQ # gatherSyncIQTargetRecords: Gather incoming SyncIQ info from record files sub gatherSyncIQTargetRecords { if (wasCalled()) { return } # Setup my %SECTIONS = ('id' => 'ID', 'policy-name' => 'Name', 'src-cluster-name' => 'Source', 'job-status' => 'Last Job State', 'job-status-timestamp' => 'Timestamp'); my @SECTIONS = keys %SECTIONS; my @FILES_LIVE = ('/ifs/.ifsvar/modules/tsm/config/target_record.xml', '/ifs/.ifsvar/modules/tsm/config/target_records/*.xml'); my @FILES_LOGS = ('local/ifsvar_modules.tar/modules/tsm/config/target_record.xml', 'local/ifsvar_modules.tar/modules/tsm/config/target_records/*.xml'); local $"='|'; my $CMD_GREP = qq[egrep '<(@SECTIONS)>']; my $RE_PID = qr/([^<]*)([^<]*)&1 $CMD_GREP @FILES_LIVE]; my $CMD_LOGS = qq[2>&1 $CMD_GREP @FILES_LOGS]; # Gather my $gather = gatherLocal(isLive()?$CMD_LIVE:$CMD_LOGS); # Parse my %policies = (); my $pid = ''; foreach my $line (@$gather) { if ($line =~ $RE_PID) { $pid = $1 } if (!$pid) { next } if ($line =~ $RE_SECTION) { $policies{$pid}{$SECTIONS{$1}} = $2 } } # Store foreach my $pid (keys %policies) { my $policy = $policies{$pid}; # Filter out policies missing details or 90 days since last ran if ($policy->{'Name'} and $policy->{'Source'} and $policy->{'Last Job State'} and $policy->{'Timestamp'} and $RPS{'Timestamp'} - $policy->{'Timestamp'} <= 7776000) { $CLUSTER_INFO{'SyncIQ'}{'Target'}{$pid} = $policy; debug("CLUSTER_INFO->SyncIQ->Target->${pid}=".toJSON($policy)); } } } # End gatherSyncIQTargetRecords ### # addCustomerInformation: Adds information to be provided to the customer # while attempting to ensure information is unique. sub addCustomerInformation { my ($section, @info) = @_; if (!$section) { return } if (@info) { push(@{$RPS{'Customer Information'}{$section}}, @info); } else { push(@{$RPS{'Customer Information'}{'Generic'}}, $section); } } # End addCustomerInformation sub addReleaseNotes { # OneFS Release Notes my $MSG_RNS = 'Review the OneFS release notes for each major OneFS version'. ' between the current and destination OneFS versions prior to'. ' upgrading.'; my $MSG_DSP = "Drive Support Package Release Notes: $ReleaseNotes{'DF'}"; my $SECTION = 'Release Notes'; my @rns = (); foreach my $version (sort {compareVersions($b, $a)} keys %{$ReleaseNotes{'OneFS'}}) { if (isCurrentOneFS("<$version") and isDestinationOneFS(">=$version")) { push(@rns, "OneFS $version Release Notes: ". "$ReleaseNotes{'OneFS'}{$version}"); } } if (@rns) { addCustomerInformation($SECTION, $MSG_RNS, @rns) } # Patch README if ($RPS{'Post Upgrade Patches'} or $RPS{'Pre Upgrade Patches'}) { my $version = isDestinationOneFS() || isCurrentOneFS(); my $major = ''; if ($version =~ /^(^\d+\.\d+\.\d+)\./) { $major = $1 } if ($ReleaseNotes{'RUP'}{$major}) { addCustomerInformation($SECTION, "OneFS $major RUP README:". " $ReleaseNotes{'RUP'}{$major}"); } } # Drive Support Package Release Notes if ($RPS{'Drive Support Package'}{'Target'}) { addCustomerInformation($SECTION, $MSG_DSP); } # Node Firmware Package Release Notes if ($RPS{'Node Firmware'}{'Target'}) { my $major = ''; if ($RPS{'IsiFw'} =~ /^(\d+)\./) { $major = $1 } foreach my $version (keys %{$ReleaseNotes{'NF'}}) { if (compareVersions($version, $major) == 0) { addCustomerInformation($SECTION, "Node Firmware Package $version Release". " Notes: $ReleaseNotes{'NF'}{$version}"); } } } } ### # addRequiredFiles: Adds files to the list of required files, making # sure to merge with existing files to avoid any duplicates sub addRequiredFiles { my @files = @_; if (!@files) { return } if (!$RPS{'Files'}) { @{$RPS{'Files'}} = () } @{$RPS{'Files'}} = merge(@{$RPS{'Files'}}, @files); } # End addRequiredFiles ### # getActivitiesName: Returns an array of activity names that # corresponds to activities in the RCM Scheduler based on the # activity type ( Drive Firmware/ Node Firmware/ Rolling # Upgrade/ Rolling Reboot/ FCO F042415EE ) and a number of # nodes or for Isilon Hourly activities the duration. sub getActivitiesName { my ($self) = @_; if (!$self) { return } our %Activities; my $type = $self->{'Type'}; if ($type eq 'Node Firmware Update' or $type eq 'OneFS Upgrade' or $type eq 'Drive Support Package' or $type eq 'Patch' or $type eq 'Reboot') { $type = 'Isilon Hourly'; } elsif ($type eq 'None') { $type = $self->{'Description'}; } if (!$Activities{$type}) { return ($type); } my $count = $self->{'Count'}; if ($type eq 'Isilon Hourly' and $self->{'Duration'}) { $count = $self->{'Duration'} / 60; } if (!$count) { return } my $last_index = 0; my @activities = (); foreach my $index (sort {$b <=> $a} keys %{$Activities{$type}}) { if ($count <= 0) { last } if ($count > $index) { if ($last_index) { $index = $last_index } push(@activities, $Activities{$type}{$index}); $count = $count - $index; $last_index = 0; redo; } if ($count == $index) { push(@activities, $Activities{$type}{$index}); $last_index = 0; last; } $last_index = $index; } if ($last_index) { push(@activities, $Activities{$type}{$last_index}); } return @activities; } # End getActivitiesName # getActivityDuration: Return the estimated duration for this activity sub getActivityDuration { my ($self) = @_; if (!$self) { return 0 } if (!$self->{'Type'}) { return 0 } our %Durations; ### # Parallel upgrade estimates if ($self->{'Nodes'} and $self->{'Reboot'} and ($self->{'Method'} and $self->{'Method'} eq 'parallel') and ($self->{'Type'} eq 'OneFS Upgrade' or $self->{'Type'} eq 'Node Firmware Update' or $self->{'Type'} eq 'Patch')) { gatherNeighborhoods(); if ($CLUSTER_INFO{'Neighborhoods'}) { my @nodes = @{$self->{'Nodes'}}; delete $self->{'Nodes'}; $self->{'Method'} = 'rolling'; my $max_duration = 0; foreach my $n (values %{$CLUSTER_INFO{'Neighborhoods'}}) { @{$self->{'Nodes'}} = array_duplicates(@$n, @nodes); my $duration = getActivityDuration($self); if ($duration > $max_duration) { $max_duration = $duration; } } $self->{'Method'} = 'parallel'; @{$self->{'Nodes'}} = @nodes; return $max_duration; } } # Simultaneous Node Firmware estimate, 1 hour, doubles for Gen6 hardware if ($self->{'Method'} and $self->{'Method'} eq 'simultaneous' and $self->{'Nodes'} and $self->{'Type'} eq 'Node Firmware Update') { if ($CLUSTER_INFO{'Gen6'}) { my @nodes = array_duplicates(@{$CLUSTER_INFO{'Gen6'}}, @{$self->{'Nodes'}}); if (@nodes) { return 120 } } return 60; } # Activities defined as 'Isilon Hourly' have a preset count which is the # number of hours associated with the activity if ($self->{'Type'} eq 'Isilon Hourly') { return $self->{'Count'} * 60; } # Start calculating the duration my $duration = 0; # Add time for the base duration if ($Durations{$self->{'Type'}}{'Base'}) { $duration += $Durations{$self->{'Type'}}{'Base'}; } # Add time for the node duration if ($self->{'Method'} and $self->{'Nodes'} and $self->{'Reboot'} and $Durations{$self->{'Type'}}{'Nodes'}) { my $dur = $Durations{$self->{'Type'}}{'Nodes'}; if ($self->{'Method'} eq 'simultaneous') { $duration += ($dur * 1.5); } else { $duration += ($dur * scalar(@{$self->{'Nodes'}})); } } # Add time based on hardware type for reboots if ($self->{'Reboot'} and $self->{'Nodes'} and $self->{'Method'}) { my $href_dur = $Durations{'Reboot'}; my $max_duration = 0; my $total_duration = 0; foreach my $lnn (@{$self->{'Nodes'}}) { if (!$NODE_INFO{$lnn}{'Product'}) { next } my $prod = $NODE_INFO{$lnn}{'Product'}; my ($match) = grep {$prod =~ /$_/} keys %{$href_dur}; if (!$match) { next } $total_duration += $href_dur->{$match}; if ($href_dur->{$match} > $max_duration) { $max_duration = $href_dur->{$match}; } } if ($self->{'Method'} eq 'simultaneous') { $duration += $max_duration; } else { $duration += $total_duration; } } # Add time per drive for Drive Support Package if ($self->{'Reference'} and $self->{'Reference'}{'Drives'}) { my $dur = $Durations{'Drive Support Package'}{'Drives'}; my $drives = $self->{'Reference'}{'Drives'}; $duration += ($drives * $dur); } # Add time for service restarts on rebootless rolling patch installs if ($self->{'Type'} eq 'Patch' and !$self->{'Reboot'}) { my $dur = $Durations{'Patch'}{'Service Restart'}; if ($self->{'Method'} and $self->{'Method'} eq 'simultaneous') { $duration += ($dur * 1.5); } else { $duration += ($dur * scalar(@{$self->{'Nodes'}})); } } # Add time based on the device types for Node Firmware Updates if ($self->{'Reference'} and $self->{'Reference'}{'Devices'}) { $self->{'Devices'} = $self->{'Reference'}{'Devices'}; } if ($self->{'Nodes'} and $self->{'Devices'}) { my $href_dur = $Durations{'Node Firmware Update'}; foreach my $dev (keys %{$self->{'Devices'}}) { my ($match) = grep {$dev =~ /$_/} keys %{$href_dur}; if (!$match) { next } my @nodes = array_duplicates(@{$self->{'Nodes'}}, @{$self->{'Devices'}{$dev}}); $duration += ($href_dur->{$match} * scalar(@nodes)); } } return $duration; } # End getActivityDuration ### # isUpgradeRestripe: Provided a source and destination version, returns 1 if # an upgrade job will run after the upgrade sub isUpgradeRestripe { my ($source, $destination) = @_; if (!$source or !$destination) { debug('isUpgradeRestripe: Not enough arguments provided.'); return 0; } debug("isUpgradeRestripe: Source: $source, Destination: $destination"); # List of versions where upgrade jobs run after upgrade my @upgrade_restripe = ( '6', '7', '8', '9.2' ); # Return 1 if the source to destination upgrade starts an upgrade job foreach my $version (@upgrade_restripe) { if (compareVersions($source, $version) < 0 and compareVersions($destination, $version) >= 0) { return 1; } } # Return 0 if it will not return 0; } # End isUpgradeRestripe ### # isUpgradeSimultaneous: Provided a source and target version, returns 1 if upgrade must be simultaneous sub isUpgradeSimultaneous { if (!(defined $_[0] and defined $_[1])) { debug("isUpgradeSimultaneous: Not enough arguments provided."); return 0; } debug("isUpgradeSimultaneous: $_[0] $_[1]"); return 1 if (compareVersions($_[0], "5") == 0 and compareVersions($_[1], "6") >= 0); return 1 if (compareVersions($_[0], "6.0") == 0 and compareVersions($_[1], "6.5") >= 0); return 1 if (compareVersions($_[0], "6.5") == 0 and compareVersions($_[1], "7") >= 0); return 1 if (compareVersions($_[0], "7.0") == 0 and compareVersions($_[1], "7.1") >= 0); return 1 if (compareVersions($_[0], "7") == 0 and compareVersions($_[1], "8") >= 0); return 1 if ($_[0] =~ /^7\.1\.1\.[89]$/ and $_[1] =~ /^7\.2\.1\.3$/); return 0; } # End isUpgradeSimultaneous ### # patchSort: Sorting routine to allow patch ordering when a patch has a priority sub patchSort { my @unsorted = @_; # Sort first by priority foreach my $a (0..$#unsorted) { foreach my $b (0..$#unsorted) { next unless $a < $b; if (($RPS{$_}{$unsorted[$a]}{'Priority'} and $RPS{$_}{$unsorted[$b]}{'Priority'} and $RPS{$_}{$unsorted[$a]}{'Priority'} < $RPS{$_}{$unsorted[$b]}{'Priority'}) or (!$RPS{$_}{$unsorted[$a]}{'Priority'} and $RPS{$_}{$unsorted[$b]}{'Priority'} and 0 < $RPS{$_}{$unsorted[$b]}{'Priority'}) or ($RPS{$_}{$unsorted[$a]}{'Priority'} and !$RPS{$_}{$unsorted[$b]}{'Priority'} and $RPS{$_}{$unsorted[$a]}{'Priority'} < 0)) { my $tmp = $unsorted[$a]; $unsorted[$a] = $unsorted[$b]; $unsorted[$b] = $tmp; } } } # Adjust for any pre-requisites foreach my $a (0..$#unsorted) { foreach my $b (0..$#unsorted) { if ($RPS{$_}{$unsorted[$b]}{'Requires'}) { foreach my $i ($b..$#unsorted) { next unless $i > $b; foreach my $r (@{$RPS{$_}{$unsorted[$b]}{'Requires'}}) { next unless $unsorted[$i] eq $r; my $tmp = splice(@unsorted, $i, 1); splice(@unsorted, $b, 0, $tmp); } } } } } return @unsorted } # End patchSort ### # printCustomerInformation: Prints gathered customer information sub printCustomerInformation { if (!$RPS{'Customer Information'}) { return } my @info = (); push(@info, ''); push(@info, ''); push(@info, colored('The following information should be included to the customer:', 'bold underline')); push(@info, ''); if ($RPS{'Customer Information'}{'Generic'}) { push(@info, @{$RPS{'Customer Information'}{'Generic'}}); } foreach my $header (keys %{$RPS{'Customer Information'}}) { if ($header eq 'Generic') { next } push(@info, '-'x20); push(@info, $header); push(@info, '-'x20); push(@info, @{$RPS{'Customer Information'}{$header}}); } push(@info, ''); # Print the customer information local $, = "\n"; print @info; } # End printCustomerInformation ### # printRequiredFiles: Prints the list of files required sub printRequiredFiles { if (!$RPS{'Files'}) { return } our %CodeLinks; our %Bundles; my @files = @{$RPS{'Files'}}; # Remove files that are included in a bundle my @bundles = grep {$_ =~ /bundle/i} @files; foreach my $bundle (@bundles) { foreach my $file (@{$Bundles{$bundle}}) { @files = grep {$_ ne $file} @files; } } # Build the output to print my @output = (); push(@output, ''); push(@output, ''); push(@output, colored('The following files will need to be downloaded to /ifs/data/:', 'bold underline')); push(@output, ''); foreach my $file (sort @files) { push(@output, $file); if ($CodeLinks{$file}{'Code'}){ push(@output, "Code: $CodeLinks{$file}{'Code'}"); } if ($CodeLinks{$file}{'sha256'}) { push(@output, "sha256: $CodeLinks{$file}{'sha256'}"); } push(@output, ''); } # Print the files local $, = "\n"; print @output; } # End printRequiredFiles ### # printUpgradePlan: Prints out the upgrade plan sub printUpgradePlan { my ($aref_plan) = @_; if (!$aref_plan or !@$aref_plan) { return } # Determine the OneFS destination my $destination = '0.0.0.0'; foreach my $href_act (@$aref_plan) { if (!$href_act->{'Target'} or $href_act->{'Target'} !~ /^\d+\.\d+\.\d+\.\d+$/) { next } if (compareVersions($destination, $href_act->{'Target'}) < 0) { $destination = $href_act->{'Target'}; } } my @output = (); push(@output, ''); push(@output, ''); push(@output, colored('The following upgrade plan is for scheduling activities to be performed by Remote Proactive:', 'bold underline')); push(@output, ''); push(@output, 'System Details:'); push(@output, "Cluster Name: $CLUSTER_INFO{'Name'}"); push(@output, "Number of Nodes: $CLUSTER_INFO{'Node Count'}"); if ($CLUSTER_INFO{'SyncIQ'}{'Source'} || $CLUSTER_INFO{'SyncIQ'}{'Target'}) { push(@output, 'Replication Status (SyncIQ): Yes'); } else { push(@output, 'Replication Status (SyncIQ): No'); } if (isCompliance()) { push(@output, 'Compliance Mode: Yes'); } else { push(@output, 'Compliance Mode: No'); } push(@output, "Current OneFS: $CLUSTER_INFO{'OneFS'}"); if ($destination ne '0.0.0.0') { push(@output, "Destination OneFS: $destination"); } else { push(@output, 'Destination OneFS: No OneFS Change'); } push(@output, ''); push(@output, 'Schedule the following activities:'); my $activity_number = 0; if ($RPS{'AutoTC'}) { if ($RPS{'Requires TC'}) { $activity_number++; push(@output, "$activity_number: Technical Consultation"); } elsif ($RPS{'Issues'}) { $activity_number++; push(@output, "$activity_number: Triage Event"); } } foreach my $href_act (@$aref_plan) { next unless $href_act->{'Type'}; my @activities = getActivitiesName($href_act); foreach my $activity_name (@activities) { $activity_number++; if ($activity_number > 1) { push(@output, '') } push(@output, "$activity_number: $activity_name"); if ($href_act->{'Type'} eq 'None') { next } push(@output, "Current Code: $href_act->{'Current Code'}"); if ($href_act->{'Target'}) { push(@output, "Upgrading To: $href_act->{'Target'}"); } else { push(@output, 'Upgrading To: No OneFS Change'); } if ($href_act->{'Patches'}) { local $" = ', '; push(@output, "Patch: @{$href_act->{'Patches'}}"); } if ($href_act->{'Additional Comments'}) { if (scalar(@{$href_act->{'Additional Comments'}}) == 1) { push(@output, "Additional Comments: $href_act->{'Additional Comments'}[0]"); } else { local $" = "\n# "; push(@output, "Additional Comments: \n# @{$href_act->{'Additional Comments'}}"); } } } } push(@output, ''); # Print the upgrade plan local $, = "\n"; print @output; } # End printUpgradePlan ### # removeRequiredFiles: Remove files from the list of required files sub removeRequiredFiles { my @files = @_; if (!@files or !$RPS{'Files'}) { return } foreach my $file (@files) { @{$RPS{'Files'}} = grep {$_ ne $file} @{$RPS{'Files'}}; } } # End removeRequiredFiles ### # UPG_F011018EE: Upgrade plan generator for FCO F011018EE sub UPG_F011018EE { my ($aref_plan, $type) = @_; if (!$aref_plan or !$type) { return } my %activity = ( 'Type' => 'Isilon Hourly', 'Current Code' => UPG_getCurrentCode($aref_plan), 'Standalone' => 1, 'Count' => 1, ); if ($type eq 'F011018EE1') { $activity{'Description'} = 'FCO F011018EE Part 1 of 3'; @{$activity{'Additional Comments'}} = ('Start IntegrityScan job'); } elsif ($type eq 'F011018EE2') { $activity{'Description'} = 'FCO F011018EE Part 2 of 3'; @{$activity{'Additional Comments'}} = ('Review IntegrityScan job results, modify sysctl setting,'. ' and start Collect job'); } elsif ($type eq 'F011018EE3') { $activity{'Description'} = 'FCO F011018EE Part 3 of 3'; @{$activity{'Additional Comments'}} = ('Review Collect job results and restore sysctl setting'); } $activity{'Duration'} = getActivityDuration(\%activity); push(@$aref_plan, \%activity); } # End UPG_F011018EE ### # UPG_F042415EE: The upgrade plan generator for FCO F042415EE sub UPG_F042415EE { my ($aref_plan, $type) = @_; if (!$aref_plan or !$type) { return } my $href_details = $RPS{$type}; my @nodes = @{$href_details->{'Nodes'}}; my @nodes_with_l3 = (); foreach my $lnn (@nodes) { if ($NODE_INFO{$lnn}{'L3'}) { push(@nodes_with_l3, $lnn); } else { my %activity = ( 'Reference' => $RPS{$type}, 'Type' => $type, 'Current Code' => UPG_getCurrentCode($aref_plan), 'Target' => 'Drive Firmware Package 1.10', 'Standalone' => 1, 'Count' => 1, 'Additional Comments' => ["Node $lnn"], ); push(@$aref_plan, \%activity); } } if (@nodes_with_l3) { my $count = scalar(@nodes_with_l3); my %activity = ( 'Reference' => $RPS{$type}, 'Type' => $type, 'Current Code' => UPG_getCurrentCode($aref_plan), 'Target' => 'Drive Firmware Package 1.10', 'Standalone' => 1, 'Count' => $count, ); if (scalar(@nodes) != $count or scalar(@nodes) != $CLUSTER_INFO{'Node Count'}) { $activity{'Additional Comments'} = [ 'Nodes '.compressRange(@nodes_with_l3) ]; } push(@$aref_plan, \%activity); } addRequiredFiles('Isilon Drive Firmware Package 1.10'); } # End UPG_F042415EE ### # UPG_generic: Generate activities for most things sub UPG_generic { my ($aref_plan, $type) = @_; if (!$aref_plan or !$type or !$RPS{$type}) { return } my $href_details = $RPS{$type}; ### # Create the activity my %activity = (); $activity{'Reference'} = $RPS{$type}; # Activity current code $activity{'Current Code'} = UPG_getCurrentCode($aref_plan); # Activity type, description, and method $activity{'Type'} = $type; if ($type eq 'Drive Firmware Reboot' or $type eq 'Proactive Reboot') { $activity{'Reboot'} = 1; $activity{'Type'} = 'Reboot'; if ($RPS{'Simultaneous'}) { $activity{'Method'} = 'simultaneous'; } else { $activity{'Method'} = 'rolling'; } } elsif ($type eq 'BMC Firmware' or $type eq 'Node Firmware') { $activity{'Type'} = 'Node Firmware Update'; $activity{'Reboot'} = 1; if ($href_details->{'Method'} and $href_details->{'Method'} eq 'parallel' and compareVersions($activity{'Current Code'}, '8.2') >= 0) { $activity{'Method'} = 'parallel'; if (compareVersions($activity{'Current Code'}, '9.2') >= 0) { $activity{'Description'} = 'Node Firmware Update (Reboot)(Parallel)'; push(@{$activity{'Additional Comments'}}, 'Use the additional argument --parallel'); } elsif (compareVersions($activity{'Current Code'}, '9') >= 0) { $activity{'Description'} = 'Node Firmware Update (Reboot)(Balanced)'; push(@{$activity{'Additional Comments'}}, 'Perform a "Balanced" node firmware update'); } elsif (compareVersions($activity{'Current Code'}, '8.2') >= 0) { $activity{'Description'} = 'Node Firmware Update (Reboot)(Balanced)'; push(@{$activity{'Additional Comments'}}, 'Use isi_upgrade_helper to perform a "Balanced" node'. ' firmware update'); } } elsif ($href_details->{'Method'} and $href_details->{'Method'} eq 'simultaneous' and compareVersions($activity{'Current Code'}, '8.2') >= 0) { $activity{'Method'} = 'simultaneous'; if (compareVersions($activity{'Current Code'}, '9.2') >= 0) { $activity{'Description'} = 'Node Firmware Update (Reboot)(Simultaneous)'; push(@{$activity{'Additional Comments'}}, 'Use the additional argument --simultaneous'); } elsif (compareVersions($activity{'Current Code'}, '9') >= 0) { $activity{'Description'} = 'Node Firmware Update (Reboot)(Fastest)'; push(@{$activity{'Additional Comments'}}, 'Perform a "Fastest" node firmware update'); } elsif (compareVersions($activity{'Current Code'}, '8.2') >= 0) { $activity{'Description'} = 'Node Firmware Update (Reboot)(Fastest)'; push(@{$activity{'Additional Comments'}}, 'Use isi_upgrade_helper to perform a "Fastest" node'. ' firmware update'); } } else { $activity{'Method'} = 'rolling'; $activity{'Description'} = 'Node Firmware Update (Reboot)(Rolling)'; if ($type eq 'Node Firmware' and $href_details->{'Newer'} and compareVersions($activity{'Current Code'}, '8') >= 0 and compareVersions($activity{'Current Code'}, '9') < 0) { push(@{$activity{'Additional Comments'}}, 'To avoid applying firmware updates to nodes that are'. ' already at newer firmware levels, use the following'. ' additional argument: --nodes-to-upgrade='. compressRange(@{$href_details->{'Nodes'}})); } } } elsif ($type =~ /Drive Support Package/) { $activity{'Description'} = 'Non-Disruptive Drive Firmware Update'; } # Activity destination code if ($href_details->{'Target'}) { $activity{'Target'} = $href_details->{'Target'}; } # Activity patches if ($href_details->{'Patch'}) { push(@{$activity{'Patches'}}, keys %{$href_details->{'Patch'}}); } # Activity nodes if ($href_details->{'Nodes'}) { $activity{'Nodes'} = $href_details->{'Nodes'}; } else { $activity{'Nodes'} = \@LNNS; } # Activity count $activity{'Count'} = scalar(@{$activity{'Nodes'}}); # Activity comments if ($href_details->{'Additional Comments'}) { push(@{$activity{'Additional Comments'}}, @{$href_details->{'Additional Comments'}}); } # Activity duration $activity{'Duration'} = getActivityDuration(\%activity); # End creating activity ### # Add to plan push(@$aref_plan, \%activity); # For any priority updates, provide the customer details if ($href_details->{'Priority'}) { addCustomerInformation('Upgrade Plan', @{$href_details->{'Priority'}}); } # Add required files and patch impacts if ($href_details->{'Patch'}) { foreach my $patch (keys %{$href_details->{'Patch'}}) { if ($patch !~ /uninstall/) { addRequiredFiles($patch) } my $href_patch = $href_details->{'Patch'}{$patch}; if ($href_patch->{'Impacts'}) { addCustomerInformation('Upgrade Plan', $href_patch->{'Impacts'}); } if ($href_patch->{'Important'}) { addCustomerInformation('Upgrade Plan', $href_patch->{'Important'}); } } } # Add required files for the activity if ($href_details->{'Files'}) { addRequiredFiles(@{$href_details->{'Files'}}); } } # UPG_generic sub UPG_getCurrentCode { my ($aref_plan) = @_; if (!$aref_plan or !@$aref_plan) { return $CLUSTER_INFO{'OneFS'}; } if ($aref_plan->[-1]{'Target'} and $aref_plan->[-1]{'Target'} =~ /\d+\.\d+\.\d+\.\d+/) { return $aref_plan->[-1]{'Target'}; } return $aref_plan->[-1]{'Current Code'}; } ### # UPG_OneFS: Generate OneFS upgrade activities and adds them to the upgrade plan sub UPG_OneFS { my ($aref_upgrade_plan, $upgrade_step) = @_; if (!$aref_upgrade_plan or !$upgrade_step) { return } $current_code = UPG_getCurrentCode($aref_plan); my @destinations = @{$RPS{'Upgrade Path'}}[1..$#{$RPS{'Upgrade Path'}}]; foreach my $destination (@destinations) { my %activity = ( 'Reference' => $RPS{'OneFS'}, 'Current Code' => $current_code, 'Target' => $destination, 'Count' => $CLUSTER_INFO{'Node Count'}, 'Nodes' => \@LNNS, 'Type' => 'OneFS Upgrade', 'Description' => 'OneFS Upgrade', 'Reboot' => 1, ); # Include node firmware if ($RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == 'OneFS' and compareVersions($current_code, '9.2') >= 0) { $activity{'Devices'} = $RPS{'Node Firmware'}{'Devices'}; push(@{$activity{'Additional Comments'}}, 'Include node firmware updates during the OneFS upgrade by using'. ' --fw-pkg='); $RPS{'Node Firmware'}{'Position'} = 'Included'; $activity{'Description'} .= ' with Node Firmware Updates'; } if (isUpgradeSimultaneous($current_code, $destination) or ($RPS{'OneFS'}{'Method'} and $RPS{'OneFS'}{'Method'} eq 'simultaneous')) { $activity{'Description'} .= ' (Reboot)(Simultaneous)'; $activity{'Method'} = 'simultaneous'; if (compareVersions($current_code, '8') >= 0) { push(@{$activity{'Additional Comments'}}, 'Perform OneFS upgrade with --simultaneous'); } } elsif ($RPS{'OneFS'}{'Method'} and $RPS{'OneFS'}{'Method'} eq 'parallel' and compareVersions($current_code, '8.2.2') >= 0) { $activity{'Description'} .= ' (Reboot)(Parallel)'; $activity{'Method'} = 'parallel'; push(@{$activity{'Additional Comments'}}, 'Perform OneFS upgrade with --parallel'); } else { $activity{'Description'} .= ' (Reboot)(Rolling)'; $activity{'Method'} = 'rolling'; if (compareVersions($current_code, '8') < 0 or compareVersions($current_code, '8.2.2') >=0) { push(@{$activity{'Additional Comments'}}, 'Perform OneFS upgrade with --rolling'); } } $activity{'Duration'} = getActivityDuration(\%activity); if ($Bundles{"Isilon OneFS $destination installation bundle"} and (!$Patched_OneFS{$destination} or !$RPS{'Disable Pre-Patched'})) { addRequiredFiles("Isilon OneFS $destination installation bundle"); } else { addRequiredFiles("Isilon OneFS $destination installation file"); } if (isDestinationOneFS($destination) and $RPS{'Post Upgrade Patches'} or !isDestinationOneFS($destination)) { if ($Patched_OneFS{$destination} and !$RPS{'Disable Pre-Patched'}) { # Upgrading to pre-patched OneFS image # Compare post upgrade patch list to those included in the # pre-patched image and then delete from the post upgrade # patches list the included patches and any PATCH-INFRA patches my @included_patches = @{$Patched_OneFS{$destination}}; my @post_upgrade_patches = keys %{$RPS{'Post Upgrade Patches'}}; foreach my $patch (@post_upgrade_patches) { my ($included) = grep {$_ eq $patch} @included_patches; if ($included) { delete $RPS{'Post Upgrade Patches'}{$patch}; } # Remove deprecated PATCH-INFRA patches if ($patch eq '8.1.2.0_UGA-PATCH-INFRA_2019-09_255624' or $patch eq '8.2.0_UGA-PATCH-INFRA_2019-08_251917') { delete $RPS{'Post Upgrade Patches'}{$patch}; } } # Set the patches list to those included in the patched image @{$activity{'Patches'}} = @{$Patched_OneFS{$destination}}; push(@{$activity{'Additional Comments'}}, 'All patches pre-loaded in OneFS installation file'); } elsif (isDestinationOneFS($destination)) { # Standard upgrade with patches applied post upgrade push(@{$activity{'Additional Comments'}}, 'Subsequent patch installs should be scheduled in the same maintenance window as the OneFS upgrade'); addCustomerInformation('Upgrade Plan', 'Patch installs are recommended to be performed in the same maintenance window as the OneFS upgrade'); } } # Add the OneFS upgrade to the plan push(@$aref_upgrade_plan, \%activity); if (isDestinationOneFS($destination) and isDestinationOneFS('>=9.1')) { if (!$RPS{'Node Firmware'}{'Position'} or $RPS{'Node Firmware'}{'Position'} ne 'Included') { push(@$aref_upgrade_plan, { 'Reference' => $RPS{'OneFS'}, 'Type' => 'Patch', 'Description' => 'Node Firmware Assessment', 'Current Code' => $destination, 'Duration' => 15, }); } push(@$aref_upgrade_plan, { 'Reference' => $RPS{'OneFS'}, 'Type' => 'Patch', 'Description' => 'Drive Support Package Install', 'Current Code' => $destination, 'Duration' => 15, }); } # Additional steps required after the OneFS upgrade will prevent # back-to-back OneFS upgrades from being performed # Upgrade restriping job that runs post upgrade if (!isDestinationOneFS($destination) and isUpgradeRestripe($current_code, $destination)) { my %activity = ( 'Reference' => $RPS{'OneFS'}, 'Type' => 'None', 'Description' => 'One week delay to allow "Upgrade" data restriping job to complete.', 'Current Code' => $destination, 'Standalone' => 1, ); push(@$aref_upgrade_plan, \%activity); } # License conversion required after 8.1 upgrade prior to next OneFS jump if (!isDestinationOneFS($destination) and compareVersions($current_code, '8.1') < 0 and compareVersions($destination, '8.1') >= 0) { my %activity = ( 'Reference' => $RPS{'OneFS'}, 'Type' => 'None', 'Description' => 'License update must be completed prior to next OneFS upgrade', 'Current Code' => $destination, 'Standalone' => 1, 'Duration' => 0, ); push(@$aref_upgrade_plan, \%activity); addCustomerInformation('Upgrade Plan', "Please note, as you are upgrading from a version prior to OneFS 8.1, and multiple OneFS upgrades are required to reach the requested destination, the upgrade plan has been altered to allow time in between OneFS upgrades for you to update the OneFS license on the cluster. The OneFS upgrade from $destination cannot be performed until you have completed the necessary license update."); if ($CLUSTER_INFO{'RemoteSupport'}{'Enabled'} and isDestinationOneFS('>=9.1')) { addCustomerInformation('Upgrade Plan', 'The SRSv2 method for remote connectivity via "isi remotesupport connectemc" commands is enabled, after completing the license update, this method must be disabled before upgrading to OneFS 9.1 or later. It is recommended to configure the SRSv3 method for remote connectivity via "isi esrs" commands or in the WebUI. See https://support.emc.com/docu95370 for more information on how to configure SRSv3.'); } } $current_code = $destination; } } # End UPG_OneFS ### # UPG_patches: The upgrade plan generator for patches sub UPG_patches { my ($aref_plan, $type) = @_; if (!$aref_plan or !$type) { return } my $href_details = $RPS{$type}; foreach my $patch_name (patchSort(keys %$href_details)) { if (!$patch_name or !defined $href_details->{$patch_name}) { next } my %patch = %{$href_details->{$patch_name}}; # Add required files if ($patch_name !~ /uninstall/i) { addRequiredFiles($patch_name) } # Add important information if ($patch{'Important'}) { addCustomerInformation($patch{'Important'}) } # Add impacts if simultaneous or no service restart script if ($patch{'Impacts'} and (($RPS{'Patch'}{'Method'} and $RPS{'Patch'}{'Method'} eq 'simultaneous') or !$patch{'Service Restart'})) { addCustomerInformation($patch{'Impacts'}); } ### # Create the activity my %activity = ( 'Reference' => $RPS{$type}, 'Patches' => [ $patch_name ], 'Type' => 'Patch', 'Current Code' => UPG_getCurrentCode($aref_plan), 'Nodes' => \@LNNS, ); # Patches that change the OneFS version if ($patch{'Target'}) { $activity{'Target'} = $patch{'Target'} } # Patches that require node reboots if ($patch{'Reboot'}) { $activity{'Reboot'} = 1; $activity{'Nodes'} = $patch{'Reboot'}; } # Set the activity method and some additional values if ($RPS{'Patch'}{'Method'} and $RPS{'Patch'}{'Method'} eq 'simultaneous') { $activity{'Method'} = 'simultaneous'; } elsif ($RPS{'Patch'}{'Method'} and $patch{'Reboot'} and $RPS{'Patch'}{'Method'} eq 'parallel' and compareVersions($activity{'Current Code'}, '9.1') >= 0) { $activity{'Method'} = 'parallel'; } elsif ($patch{'Reboot'}) { $activity{'Method'} = 'rolling'; } elsif ($patch{'Service Restart'}) { $activity{'Method'} = 'rolling'; if ($patch{'Service Restart'} == 1) { $activity{'Reboot'} = 1; } } else { $activity{'Method'} = 'simultaneous'; } # Set the activity description $activity{'Description'} = 'Patch'. (($patch_name =~ /uninstall/i)?' Uninstall':' Install'). ($activity{'Reboot'}?' (Reboot)':' (Rebootless)'). '('.ucfirst($activity{'Method'}).')'; # Add to additional comments on how the patch will be applied if (compareVersions($activity{'Current Code'}, '9.1') >= 0) { $activity{'Additional Comments'} = [ 'Use the additional argument --'.$activity{'Method'} ]; } elsif ($activity{'Method'} eq 'simultaneous') { if (compareVersions($activity{'Current Code'}, '8.2') >= 0) { $activity{'Additional Comments'} = [ 'Use the additional argument --simultaneous' ]; } elsif (compareVersions($activity{'Current Code'}, '8') >= 0) { $activity{'Additional Comments'} = [ 'Use the additional argument --rolling=false' ]; } } # Add that only some nodes need to be rebooted if ($patch{'Reboot'} and $patch{'Nodes'} and scalar(@{$patch{'Nodes'}}) < $CLUSTER_INFO{'Node Count'}) { push(@{$activity{'Additional Comments'}}, 'Only the following nodes require rebooting: '. compressRange(@{$patch{'Nodes'}})); } if ($patch{'Additional Comments'}) { push(@{$activity{'Additional Comments'}}, @{$patch{'Additional Comments'}}); } $activity{'Duration'} = getActivityDuration(\%activity); push(@$aref_plan, \%activity); } } sub upgradePlanCombinedSimilar { my ($aref_upgrade_plan) = @_; if (!$aref_upgrade_plan) { return } my @new_plan = (); my @tmp_plan = (); my $href_previous = undef; foreach my $href_current (@$aref_upgrade_plan) { if (!$href_previous) { $href_previous = $href_current; @tmp_plan = ($href_previous); next; } if ($href_current->{'Type'} ne 'None' and ($href_previous->{'Type'} eq $href_current->{'Type'} or ($href_previous->{'Type'} eq 'OneFS Upgrade' and $href_current->{'Type'} eq 'Patch'))) { my $test_plan = upgradePlanConvertToCombined ([@tmp_plan, $href_current]); if (scalar(@$test_plan) > 1) { if (scalar(@tmp_plan) > 1) { push(@new_plan, shift(@$test_plan)); } else { push(@new_plan, $href_previous); } $href_previous = undef; redo; } else { push(@tmp_plan, $href_current); $href_previous = $href_current; } } else { if (scalar(@tmp_plan) > 1) { my $test_plan = upgradePlanConvertToCombined(\@tmp_plan); push(@new_plan, shift(@$test_plan)); } else { push(@new_plan, $href_previous); } $href_previous = undef; redo; } } if (@tmp_plan) { if (scalar(@tmp_plan) > 1) { my $test_plan = upgradePlanConvertToCombined(\@tmp_plan); push(@new_plan, shift(@$test_plan)); } else { push(@new_plan, $href_previous); } } return \@new_plan; } ### # upgradePlanConvertToCombined: Given an upgrade plan will convert it # into a plan that is hourly activities and combined sub upgradePlanConvertToCombined { my ($aref_upgrade_plan) = @_; if (!$aref_upgrade_plan) { return } my @c_plan = (); my $c_href = undef; foreach my $a_href (@$aref_upgrade_plan) { if ($a_href->{'Standalone'} or $a_href->{'Duration'} + ($c_href?$c_href->{'Duration'}:0) >= 720) { push(@c_plan, $c_href); $c_href = undef; } if ($a_href->{'Standalone'}) { push(@c_plan, $a_href); next; } if (!$c_href) { $c_href->{'Type'} = 'Isilon Hourly'; if ($a_href->{'Current Code'}) { $c_href->{'Current Code'} = $a_href->{'Current Code'}; } } else { push(@{$c_href->{'Additional Comments'}}, ''); } $c_href->{'Duration'} += $a_href->{'Duration'}; $c_href->{'Count'} = $c_href->{'Duration'} / 60; if ($a_href->{'Type'}) { my ($type) = getActivitiesName($a_href); if ($a_href->{'Description'}) { $type = $a_href->{'Description'}; } push(@{$c_href->{'Additional Comments'}}, sprintf('Activity Type: %s (%.1f hours)', $type, $a_href->{'Duration'} / 60)); } push(@{$c_href->{'Additional Comments'}}, "Current Code: $a_href->{'Current Code'}"); if ($a_href->{'Target'}) { push(@{$c_href->{'Additional Comments'}}, sprintf('Upgrading To: %s', $a_href->{'Target'})); if ($a_href->{'Target'} =~ /^\d+\.\d+\.\d+\.\d+$/ and (!$c_href->{'Target'} or compareVersions($c_href->{'Target'}, $a_href->{'Target'}) < 0)) { $c_href->{'Target'} = $a_href->{'Target'}; } } if ($a_href->{'Patches'}) { push(@{$c_href->{'Additional Comments'}}, sprintf('Patch: %s', join(', ', @{$a_href->{'Patches'}}))); } if ($a_href->{'Additional Comments'}) { if (scalar(@{$a_href->{'Additional Comments'}} == 1)) { push(@{$c_href->{'Additional Comments'}}, sprintf('Additional Details: %s', $a_href->{'Additional Comments'}[0])); } else { push(@{$c_href->{'Additional Comments'}}, join("\n# ", 'Additional Details:', @{$a_href->{'Additional Comments'}})); } } } if ($c_href) { push(@c_plan, $c_href) } return \@c_plan; } # End upgradePlanConvertToCombined ### # upgradePlanGenerateOrder: Returns the order of activites to be performed sub upgradePlanGenerateOrder { # Just incase node firmware was flagged to be performed post OneFS, do BMC firmware updates post node firmware if ($RPS{'BMC Firmware'}{'Position'} and $RPS{'BMC Firmware'}{'Position'} == 1 and $RPS{'CMC Firmware'}{'Nodes'} and $RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == 2) { $RPS{'BMC Firmware'}{'Position'} = 2; } # Adjust for combined OneFS + NF if (!$RPS{'OneFS'}{'Exclude Node Firmware'} and $RPS{'OneFS'}{'Method'} eq 'parallel' and $RPS{'Node Firmware'}{'Position'} and $RPS{'Upgrade Path'} and scalar(@{$RPS{'Upgrade Path'}}) > 1) { foreach my $version (@{$RPS{'Upgrade Path'}}) { if (isDestinationOneFS($version)) { last } if (compareVersions($version, '9.2') >= 0) { $RPS{'Node Firmware'}{'Position'} = 'OneFS'; last; } } } # Flag Node firmware updates as optional unless they are at FCO F010517EE levels if ($RPS{'F010517EE'} and !$RPS{'FCO'}{'F010517EE'}{'Nodes'}) { addCustomerInformation('Upgrade Plan', 'Node firmware updates available but already at required levels for FCO F010517EE and are considered optional.'); } # Flag Node Firmware updates as optional unless they are a priority if ($RPS{'F033018EE'} and $RPS{'Node Firmware'}{'Position'} and !$RPS{'Node Firmware'}{'Priority'}) { addCustomerInformation('Upgrade Plan', 'Node firmware updates available and included but considered optional for FCO F033018EE.'); } if ($RPS{'F082318EE'} and $RPS{'Drive Support Package'}{'Position'}) { addCustomerInformation('Upgrade Plan', 'Drive support package updates available and included but considered optional for FCO F082318EE.'); } # Define/reset the upgrade order my @upgrade_order = (); push(@upgrade_order, 'Drive Support Package') if ($RPS{'Drive Support Package'}{'Position'} and $RPS{'Drive Support Package'}{'Position'} == '1'); push(@upgrade_order, 'Drive Firmware Reboot') if ($RPS{'Drive Firmware Reboot'}{'Position'} and $RPS{'Drive Firmware Reboot'}{'Position'} == '1'); push(@upgrade_order, 'Proactive Reboot') if ($RPS{'Proactive Reboot'}); push(@upgrade_order, 'Pre Upgrade Patches') if ($RPS{'Pre Upgrade Patches'}); push(@upgrade_order, 'Node Firmware') if ($RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == '1'); push(@upgrade_order, 'BMC Firmware') if ($RPS{'BMC Firmware'}{'Position'} and $RPS{'BMC Firmware'}{'Position'} == '1'); push(@upgrade_order, 'Drive Support Package') if ($RPS{'Drive Support Package'}{'Position'} and $RPS{'Drive Support Package'}{'Position'} == '2'); push(@upgrade_order, 'Drive Firmware Reboot') if ($RPS{'Drive Firmware Reboot'}{'Position'} and $RPS{'Drive Firmware Reboot'}{'Position'} == '2'); push(@upgrade_order, 'OneFS') if ($RPS{'Upgrade Path'} and scalar(@{$RPS{'Upgrade Path'}}) > 1); push(@upgrade_order, 'Post Upgrade Patches') if ($RPS{'Post Upgrade Patches'}); push(@upgrade_order, 'Node Firmware') if ($RPS{'Node Firmware'}{'Position'} and $RPS{'Node Firmware'}{'Position'} == '2'); push(@upgrade_order, 'BMC Firmware') if ($RPS{'BMC Firmware'}{'Position'} and $RPS{'BMC Firmware'}{'Position'} == '2'); push(@upgrade_order, 'Drive Support Package') if ($RPS{'Drive Support Package'}{'Position'} and $RPS{'Drive Support Package'}{'Position'} == '3'); push(@upgrade_order, 'Drive Firmware Reboot') if ($RPS{'Drive Firmware Reboot'}{'Position'} and $RPS{'Drive Firmware Reboot'}{'Position'} == '3'); push(@upgrade_order, 'FCO F042415EE') if ($RPS{'FCO F042415EE'}{'Nodes'}); # Only Node Firmware updates for FCO F121917EE if ($RPS{'F121917EE'}) { @upgrade_order = (); @upgrade_order = ('Node Firmware') if $RPS{'FCO'}{'F121917EE'}{'Nodes'} and $RPS{'Node Firmware'}{'Position'}; } # Special plan for FCO F011018EE if ($RPS{'F011018EE'}) { if ($RPS{'FCO'}{'F011018EE'}{'Nodes'} and $RPS{'Node Firmware'}{'Position'}) { @upgrade_order = ('Node Firmware', 'F011018EE2', 'F011018EE3'); } else { @upgrade_order = ('F011018EE1', 'F011018EE2', 'F011018EE3'); } } # If requirements for FCO F033018EE are met already, no upgrade plan is provided. if ($RPS{'F033018EE'}) { my @results = grep {$_ =~ /Drive Support|OneFS/} @upgrade_order; if (!@results and isCurrentOneFS('>=8.1.2.0')) { @upgrade_order = (); } } # If requirements for FCO F082318EE are met already, no upgrade plan is provided. if ($RPS{'F082318EE'}) { my @results = grep {$_ =~ /Node Firmware|OneFS/} @upgrade_order; if (!@results and isCurrentOneFS('>=8.1.0.4')) { @upgrade_order = (); } } return \@upgrade_order; } # End upgradePlanOrder ### # upgradePlanGenerateOverview: Generates an overview of the upgrade plan for customer consumption. sub upgradePlanGenerateOverview { my ($aref_plan) = @_; if (!$aref_plan or !@$aref_plan) { return } my $total_duration = 0; my $activity_number = 0; my @upgrade_overview = (); foreach my $href_activity (@$aref_plan) { my %activity = %{$href_activity}; my $description = $activity{'Description'} || $activity{'Type'}; if (!$description) { next } $activity_number++; if (@upgrade_overview) { push(@upgrade_overview, '') } if ($activity{'Duration'}) { $total_duration += $activity{'Duration'}; push(@upgrade_overview, sprintf('%s: %s (%.1f hours)', $activity_number, $description, $activity{'Duration'} / 60)); } else { push(@upgrade_overview, "$activity_number: $description"); } if ($activity{'Type'} eq 'None') { next } push(@upgrade_overview, "Current Code: $activity{'Current Code'}"); if ($activity{'Target'}) { push(@upgrade_overview, 'Upgrading To: '.$activity{'Target'}); } if ($activity{'Patches'}) { push(@upgrade_overview, 'Patch: '.join(', ', @{$activity{'Patches'}})); } if ($activity{'Nodes'}) { push(@upgrade_overview, sprintf('Nodes: %s (%s nodes)', compressRange(@{$activity{'Nodes'}}), scalar(@{$activity{'Nodes'}}))); } if ($activity{'Additional Comments'}) { if (scalar(@{$activity{'Additional Comments'}}) == 1) { push(@upgrade_overview, 'Additional Comments: '.$activity{'Additional Comments'}[0]); } else { my @comments = map { '# '.$_ } @{$activity{'Additional Comments'}}; push(@upgrade_overview, 'Additional Comments: ', @comments); } } } if ($total_duration) { push(@upgrade_overview, '', sprintf('Estimated total duration: %.1f hours', $total_duration/60)); } return \@upgrade_overview; } ### # upgradePlanGeneratePlan: Generates an upgrade plan provided the upgrade order sub upgradePlanGeneratePlan { my ($aref_upgrade_order) = @_; if (!$aref_upgrade_order) { return } my @upgrade_plan = (); foreach (@$aref_upgrade_order) { if (/Support|Firmware|Reboot/) { UPG_generic(\@upgrade_plan, $_); } elsif (/^OneFS/) { UPG_OneFS(\@upgrade_plan, $_); } elsif (/Patches/) { UPG_patches(\@upgrade_plan, $_); } elsif (/F042415EE/) { UPG_F042415EE(\@upgrade_plan, $_); } elsif (/F011018EE/) { UPG_F011018EE(\@upgrade_upan, $_); } } return \@upgrade_plan; } # End upgradePlanGeneratePlan ### # upgradePlan: Steps used to generate and print an upgrade plan sub upgradePlan { if (wasCalled()) { return } # Run a few checks, the bare minimum for an upgrade plan checkUptime(); checkNodeFirmware(); checkDriveFirmware(); checkUpgradePath(); checkPatches(); # Generate the upgrade plan my $order = upgradePlanGenerateOrder(); my $plan = upgradePlanGeneratePlan($order); my $combined_plan = upgradePlanConvertToCombined($plan); my $similar_plan = upgradePlanCombinedSimilar($plan); my $overview = upgradePlanGenerateOverview($plan); if (@$overview) { addCustomerInformation('Overview of Upgrade Steps', @$overview); } # Print the upgrade plan addReleaseNotes(); printCustomerInformation(); printRequiredFiles(); if ($RPS{'F011018EE'} or $RPS{'F121917EE'} or (scalar(@$combined_plan) == 1 and $combined_plan->[0]{'Count'} <= 12)) { printUpgradePlan($combined_plan); } else { printUpgradePlan($similar_plan); } #printUpgradePlan($plan); #printUpgradePlan($combined_plan); #printUpgradePlan($similar_plan); } # End upgradePlan package Converters; use strict; use warnings; use Exporter qw( import ); eval { require Logger }; import Logger qw( debug ); our $VERSION = v1.0.0.0; BEGIN { our @EXPORT_OK = qw ( convertDurationToSeconds convertFromNumeric convertToNumeric convertIPToNumeric ); our %EXPORT_TAGS = ( ALL => \@EXPORT_OK ); } ### # convertDurationToSeconds: Convert from a duration, i.e. 1Y1W1D1H1m1s to # 123153424 seconds sub convertDurationToSeconds { my ($duration) = @_; my $seconds = 0; if (!defined $duration) { $duration = 0 } debug("convertDurationToSeconds: Argument: $duration"); if ($duration =~ /(\d+)Y/) { $seconds += $1 * 31536000 } if ($duration =~ /(\d+)M/) { $seconds += $1 * 2592000 } if ($duration =~ /(\d+)W/) { $seconds += $1 * 604800 } if ($duration =~ /(\d+)D/) { $seconds += $1 * 86400 } if ($duration =~ /(\d+)H/) { $seconds += $1 * 3600 } if ($duration =~ /(\d+)m/) { $seconds += $1 * 60 } if ($duration =~ /(\d+)s/) { $seconds += $1 } debug("convertDurationToSeconds: Return: $seconds"); return $seconds; } # End convertDurationToSeconds ### # convertFromNumeric: Convert from a number of bytes to a count of a prefix of # bytes. I.e. 1024, K => 1KB sub convertFromNumeric { my ($bytes, $prefix) = @_; my $num = 0; my $count = 0; if (!defined $bytes) { $bytes = 0 } if (!defined $prefix) { $prefix = '' } debug("convertFromNumeric: Arguments: $bytes, $prefix"); if ($bytes !~ /^\d+$/) { $bytes = 0 } if ($prefix !~ /^[KMGTP]$/) { $prefix = '' } if ($prefix eq 'K') { $count = $bytes / 2**10 } elsif ($prefix eq 'M') { $count = $bytes / 2**20 } elsif ($prefix eq 'G') { $count = $bytes / 2**30 } elsif ($prefix eq 'T') { $count = $bytes / 2**40 } elsif ($prefix eq 'P') { $count = $bytes / 2**50 } if ($count) { $count = int($count) + 1 } if ($prefix) { $num = "$count${prefix}B" } else { $num = $count } debug("convertFromNumeric: Return: $num"); return $num; } # End convertFromNumeric ### # convertToNumeric: Convert from a count and prefix into just a number sub convertToNumeric { my ($num) = @_; my $bytes = 0; my $count = 0; my $prefix = ''; if (!defined $num) { $num = 0 } debug("convertToNumeric: Arguments: $num"); if ($num =~ /^\s*(\d*\.?\d*)\s*([KMGTP])\D*\s*$/) { ($count, $prefix) = ($1, $2); } if ($prefix eq 'K') { $bytes = $count * 2**10 } elsif ($prefix eq 'M') { $bytes = $count * 2**20 } elsif ($prefix eq 'G') { $bytes = $count * 2**30 } elsif ($prefix eq 'T') { $bytes = $count * 2**40 } elsif ($prefix eq 'P') { $bytes = $count * 2**50 } debug("convertToNumeric: Return: $bytes"); return $bytes; } # End convertToNumeric ### # convertIPToNumeric: Converts a string IP into it's bitwise equivalent sub convertIPToNumeric { my ($str_ip) = @_; my $int_ip = 0b0; my @str_octets = (); if (!defined $str_ip) { $str_ip = '' } debug("convertIPToNumeric: Arguments: $str_ip"); @str_octets = split(/\./, $str_ip); if ($#str_octets == 3) { # IPv4 IP for (my $i = 3; $i >= 0; $i--) { $int_ip += $str_octets[$i] * (2 ** ((3 - $i) * 8)); } } debug('convertIPToNumeric: Return: '.sprintf('%032b', $int_ip)); return $int_ip; } # End convertIPToNumeric package Helper; use strict; use warnings; use Exporter qw( import ); eval { require Logger }; import Logger qw( debug fail ); our $VERSION = v1.1.0.1; BEGIN { our @EXPORT_OK = qw ( compressRange expandRange compareVersions merge generateUnpackTemplate upgradeDigit toJSON findLowestCommonPath gatherCluster gatherLocal array_duplicates ); our %EXPORT_TAGS = ( ALL => \@EXPORT_OK ); } our $ALRM_TMOUT = 600; our $ALRM_FAILURE = 'TMOUT'; our $ALRM_MSG = 'Gathering command timed out'; our $FRK_FAILURE = 'FRKFAIL'; our $FRK_MSG = 'Gathering failed to create a fork to run command'; ### # expandRange: Takes a node list such as 1,3,5-6,11-132,2 and generates an # array of all nodes in the list sub expandRange { my ($range) = @_; if (!defined $range) { $range = '' } debug("Argument: $range"); # Remove white space $range =~ s/\s+//g; # Validate input format if ($range !~ /^(?:(?:\d+-\d+)|,|\d+)*$/) { debug('Invalid list format'); $range = ''; } # Expand the ranges my @values = (); my @groups = split(',', $range); foreach my $group (@groups) { if ($group =~ /^\d+-\d+$/) { my ($start, $end) = split('-', $group); if ($start > $end) { debug("Invalid range, $group, $start is greater than $end"); @values = (); last; } else { push(@values, ($start .. $end)); } } elsif ($group =~ /^\d+$/) { push(@values, $group); } } # Remove duplicates and sort the array @values = sort {$a <=> $b} merge(@values); # Return the expanded range array debug("Return: [ @values ]"); return @values; } # End expandRange ### # compressRange: Takes an array of nodes and returns a compressed range. sub compressRange { my @values = @_; debug("Arguments: [ @values ]"); # Validate all values are integers for (my $i = 0; $i <= $#values; $i++) { if (!defined $values[$i]) { debug('Undefined value detected'); @values = (); last; } $values[$i] =~ s/^\s+|\s+$//g; if ($values[$i] !~ /^\d+$/) { debug('Non-integer value detected'); @values = (); last; } } # Merge duplicates and then sort the values @values = sort {$a <=> $b} merge(@values); # Create the compressed range my $range = @values?$values[0]:''; my $start = 0; for (my $i = 1; $i <= $#values; $i++) { if ($values[$i - 1] + 1 != $values[$i]) { if ($i - 1 != $start) { $range .= "-$values[$i - 1]"; } $range .= ",$values[$i]"; $start = $i; } elsif ($#values == $i) { if ($i - 1 == $start) { $range .= ",$values[$i]"; } else { $range .= "-$values[$i]"; } } } # Return the compressed range debug("Return: $range"); return $range; } # End compressRange ### # compareVersions: Compares two OneFS versions. # $1 > $2 => Positive # $1 = $2 => 0 # $1 < $2 => Negative # If only a partial version is provided, it will match up to given criteria. # IE: 7.0 and 7.0.2 would be equivalent. sub compareVersions { my @left = split(/\.|-|\+/, $_[0]); my @right = split(/\.|-|\+/, $_[1]); my $last = ($#left > $#right)?$#right:$#left; my $comparison = 0; for (my $i = 0; $i <= $last; $i++) { if ($left[$i] =~ /^\d*$/ and $right[$i] =~ /^\d*$/) { $comparison = $left[$i] - $right[$i]; } else { $comparison = $left[$i] cmp $right[$i]; } last if ($comparison); } debug("Arguments: $_[0], $_[1] Results: $comparison"); return $comparison; } # End compareVersions ### # Given two arrays, combine them removing duplicates and return the merged array sub merge { my %hash = (); $hash{$_} = 1 foreach(@_); return keys %hash; } ### # generateUnpackTemplate: Generates an unpack template based on provided header line and header labels sub generateUnpackTemplate { debug("generateUnpackTemplate: ".join(",",@_)); if (scalar(@_) <= 1) { debug("Not enough arguments provided"); return ""; } my $unpack_template = ""; my $header_line = shift; my @indexes = (); foreach my $label (@_) { push(@indexes, index($header_line, $label)); } my $previous_index = 0; foreach my $index (sort {$a <=> $b} @indexes) { next if ($index == -1 or $index == 0); my $unpack_diff = $index - $previous_index - 1; $unpack_template .= "A${unpack_diff}x"; $previous_index = $index; } $unpack_template .= "A*"; debug($unpack_template); return $unpack_template; } # End generateUnpackTemplate ### # upgradeDigit: Returns the digit the upgrade being performed is. # # Example: 8.1.0.4, 8.1.2.0 = 3rd digit difference sub upgradeDigit { return 4 unless @_ and scalar(@_) == 2; my @left = split(/\./, $_[0]); my @right = split(/\./, $_[1]); return 4 unless scalar(@left) == 4 and scalar(@right) == 4; for (my $i = 1; $i <= 4; $i++) { return $i if $left[$i - 1] ne $right[$i - 1]; } return 4; } # End upgradeDigit ### # toJSON: Converts the first argument to JSON and returns it as a string sub toJSON { my ($obj) = @_; if (!defined $obj) { return '""' } my $type = ref($obj); # The object is not a reference if ($type eq "") { # The object is a boolean or number if ($obj eq "false" or $obj eq "true" or $obj =~ /^-?[0-9]+(\.[0-9]+)?$/) { return $obj; } # The object is a string and should be quoted else { my %esc = ("\n" => '\n', "\r" => '\r', "\t" => '\t', "\f" => '\f', "\b" => '\b', "\"" => '\"', "\\" => '\\\\', "\'" => '\\\''); $obj =~ s/([\x22\x5c\n\r\t\f\b])/$esc{$1}/g; $obj =~ s/([\x00-\x08\x0b\x0e-\x1f])/'\\u00' . unpack('H2', $1)/eg; return sprintf('"%s"', $obj); } } # The object is a reference to an ARRAY if ($type eq "ARRAY") { return sprintf('[%s]', join(', ', map { toJSON($_) } @{$obj})); } # The object is a reference to a HASH if ($type eq "HASH") { my @kvs = (); foreach my $key (keys %{$obj}) { my $k = toJSON($key); my $v = toJSON($obj->{$key}); if ($k !~ /^\".*\"$/) { $k = sprintf('"%s"', $k) } push(@kvs, sprintf('%s: %s', $k, $v)); } return sprintf('{%s}', join(', ', @kvs)); } # The object is unknown return '""'; } # End toJSON ### # stopProcessGroup: Try to safely stop all processes in a group, and # then force kill after a period of time. sub stopProcessGroup { my ($pgid) = @_; if (!$pgid) { return } $pgid = "-$pgid"; if (kill 'ZERO', $pgid) { kill 'TERM', $pgid } foreach (0..7) { if (kill 'ZERO', $pgid) { sleep 1 } else { return } } if (kill 'ZERO', $pgid) { kill 'INT', $pgid } foreach (0..7) { if (kill 'ZERO', $pgid) { sleep 1 } else { return } } if (kill 'ZERO', $pgid) { kill 'KILL', $pgid } } # End stopProcessGroup ### # gatherCluster: Runs command and parses based on it being a cluster wide command. sub gatherCluster { my ($cmd) = @_; if (!$cmd) { return } debug("Command: $cmd"); my %gather = (); eval { # Use open to fork a sub process my $child_pid = open(my $output, "-|") // die $FRK_FAILURE; if ($child_pid == 0) { # As the child, set the process group to allow for easy # killing and then execute the command and never return. setpgrp; exec($cmd); } local $SIG{ALRM} = sub { stopProcessGroup($child_pid); die $ALRM_FAILURE }; alarm $ALRM_TMOUT; while (<$output>) { if (/No such file or directory|command not found:/) { next } s/\x1b\[[0-9;]*m//g; if (/^\S+-(\d+)\/\S*?:(.*?)\s*$/) { push(@{$gather{$1}}, $2) } if (/^\S+-(\d+): (.*?)\s*$/) { push(@{$gather{$1}}, $2) } } close $output; alarm 0; }; if ($@ =~ $ALRM_FAILURE) { fail($ALRM_MSG) } if ($@ =~ $FRK_FAILURE) { fail($FRK_MSG) } if ($Logger::LEVEL <= &Logger::DEBUG) { foreach my $lnn (keys %gather) { if (!$gather{$lnn}) { next } debug("--------------------", " Node $lnn", "--------------------"); debug(@{$gather{$lnn}}); } } return wantarray?%gather:\%gather; } # End gatherCluster ### # gatherLocal: Runs command and parses based on it being a local only command. sub gatherLocal { my ($cmd) = @_; if (!$cmd) { return } debug("Command: $cmd"); my @gather = (); eval { # Use open to fork a sub process my $child_pid = open(my $output, "-|") // die $FRK_FAILURE; if ($child_pid == 0) { # As the child, set the process group to allow for easy # killing and then execute the command and never return. setpgrp; exec($cmd); } local $SIG{ALRM} = sub { stopProcessGroup($child_pid); die $ALRM_FAILURE }; alarm $ALRM_TMOUT; while (<$output>) { if (/No such file or directory|command not found:/) { next } s/\x1b\[[0-9;]*m//g; s/\s+$//; push(@gather, $_); } close $output; alarm 0; }; if ($@ =~ $ALRM_FAILURE) { fail($ALRM_MSG) } if ($@ =~ $FRK_FAILURE) { fail($FRK_MSG) } debug(@gather); return wantarray?@gather:\@gather; } # End gatherLocal ### # findLowestCommonPath: Finds the 'lowest' common path given an array of paths. # Used to give a guess at what the base directory should be for each access zone. sub findLowestCommonPath { debug("Arguments [ ".join(", ", @_)." ]"); my @lowest_common_path_array = (); foreach my $share_path (@_) { my @share_path_array = split("/", $share_path); if (!@lowest_common_path_array) { @lowest_common_path_array = @share_path_array; next; } for (my $i = 0; $i <= $#lowest_common_path_array; $i++) { if ($i > $#share_path_array or $share_path_array[$i] ne $lowest_common_path_array[$i]) { splice(@lowest_common_path_array, $i); last; } } } my $lowest_common_path = join("/", @lowest_common_path_array); debug("Results [ $lowest_common_path ]"); return $lowest_common_path; } # End findLowestCommonPath ### # array_duplicates: Returns an array of items that appear more than once sub array_duplicates { my @array = @_; my %count; foreach (@array) { $count{$_}++ } return grep { $count{$_} > 1 } keys %count; } # End array_duplicates 1; package Logger; use strict; use warnings; use Term::ANSIColor; use Exporter qw( import ); eval { require Helper }; import Helper qw( compressRange ); our $VERSION = v1.0.0.0; BEGIN { our @EXPORT_OK = qw ( ALL DEBUG VERBOSE DETAILS INFO WARN CRITICAL FAIL printUrgent printResults printCheck printSummary missing info warning fail critical details verbose debug unsupported verboseHeader disableColors isResults ); our %EXPORT_TAGS = ( ALL => \@EXPORT_OK ); } use constant { ALL => 0, DEBUG => 1, VERBOSE => 2, DETAILS => 3, INFO => 4, WARN => 5, CRITICAL => 6, FAIL => 7, }; our $LEVEL = &INFO; our $JSON = 0; our @DEBUG = (); our @DETAILS = (); our @INFO = (); our @WARN = (); our @FAIL = (); our @CRITICAL = (); our %MISSING = (); our @RESULTS = (); our $" = $, = ", "; ### # enableJSON: Enable JSON mode sub enableJSON { $JSON = 1 } ### # debug: Record debug messages, but only if enabled for debug sub debug { my @messages = @_; if ($LEVEL > &DEBUG) { return @messages } my $time = time(); my $caller = (caller(1))[3]; push(@DEBUG, "$time : $caller : $_") foreach @messages; return @messages; } # End debug ### # verbose: Record verbose messages, but only if enabled for verbose sub verbose { if ($LEVEL <= &VERBOSE) { push(@DEBUG, @_); } return @_; } # End verbose ### # details: Record details messages sub details { push(@DETAILS, @_); return @_; } # End details ### # info: Record info messages sub info { push(@INFO, @_); return @_; } # End info ### # warning: Record warn messages sub warning { push(@WARN, @_); return @_; } # End warning ### # fail: Record fail messages sub fail { push(@FAIL, @_); return @_; } # End fail ### # critical: Record critical messages sub critical { push(@CRITICAL, @_); return @_; } # End critical ### # missing: Store a list of nodes with missing information sub missing { my ($lnn) = @_; if (!$lnn) { return; } $MISSING{$lnn} = 1; return; } # End missing ### # unsupported: Used for HealthCheck Framework to report unsupported checks sub unsupported { my @reason = @_; debug(@reason); my $subroutine = (caller(1))[3]; $subroutine =~ s/^.*:://; my %result = ( 'name' => "ioca_$subroutine", 'node' => "cluster", 'status' => "UNSUPPORTED", 'value' => -1, 'parameters' => {}, 'details' => \@reason, ); push(@RESULTS, \%result); return; } # End unsupported ### # verboseHeader: A verbose header sub verboseHeader { my ($header) = @_; if (!$header) { return; } verbose('', '==============================', $header, '=============================='); return; } # End verboseHeader ### # isResults: Return 1 if there are results to print sub isResults { my ($check_level) = @_; if ($check_level) { if ($check_level eq &ALL) { return 1; } if ($check_level <= &VERBOSE and @DEBUG) { return 1; } if ($check_level <= &INFO and (@DETAILS or @INFO)) { return 1; } if ($check_level <= &WARN and @WARN) { return 1; } if ($check_level <= &FAIL and (@CRITICAL or @FAIL or %MISSING)) { return 1; } } elsif (@FAIL or @CRITICAL or @WARN or @INFO or @DETAILS or @DEBUG or %MISSING) { return 1; } return 0; } # End isResults ### # disableColors: Disables terminal coloring sub disableColors { no warnings 'redefine'; *colored = sub { if ($_[0]) { return $_[0]; } }; return; } # End disableColors ### # printUrgent: Prints a bold red urgent message sub printUrgent { return unless $_[0]; print colored(sprintf("## %-65s ##\n", $_[0]), "bold red"); return; } # End printUrgent ### # printCheck: Prints header for starting checks and, if a second # argument is provided, prints that as the result of the check. sub printCheck { printf "%-50s", $_[0] if ($_[0]); printf "%s\n", $_[1] if ($_[1]); return; } # End printCheck ### # printResults: If provided an argument, prints it followed by an end # line, otherwise evaluates variables used in checks and prints results. sub printResults { my ($msg) = @_; # If there was any nodes missing information if (%MISSING) { fail(sprintf('Missing information for the following nodes: %s', compressRange(keys %MISSING))); } # Add to count of issues if (@FAIL) { $IOCA::RPS{'Issues'}++ } if (!$JSON) { # Colors for each level of failure my $FAIL = colored(($msg)?"FAIL ($msg)":"FAIL", 'red')."\n"; my $WARN = colored(($msg)?"WARN ($msg)":"WARN", 'yellow')."\n"; my $PASS = colored(($msg)?"PASS ($msg)":"PASS", 'green')."\n"; my $INFO = colored(($msg)?"INFO ($msg)":"INFO", 'blue')."\n"; if (@FAIL) { print $FAIL; } elsif (@CRITICAL) { print $FAIL; } elsif (@WARN) { print $WARN; } elsif (@INFO) { print $INFO; } else { print $PASS; } print colored(" FAIL: ", 'red')."$_\n" foreach (@FAIL); print colored(" CRITICAL: ", 'red')."$_\n" foreach (@CRITICAL); print colored(" WARN: ", 'yellow')."$_\n" foreach (@WARN); print " DETAILS: $_\n" foreach (@DETAILS); print colored(" INFO: ", 'blue')."$_\n" foreach (@INFO); print " $_\n" foreach (@DEBUG); } my %result = (); my $subroutine = (caller(1))[3]; $subroutine =~ s/^IOCA:://; $result{'name'} = "$subroutine"; $result{'name'} = "ioca_$subroutine" if $JSON; $result{'node'} = "cluster"; %{$result{'parameters'}} = (); $result{'status'} = "OK"; $result{'value'} = 100; $result{'status'} = "WARNING" if @WARN; $result{'value'} = 75 if @WARN; $result{'status'} = "CRITICAL" if @FAIL or @CRITICAL; $result{'value'} = 50 if @FAIL or @CRITICAL; @{$result{'details'}} = (); push(@{$result{'details'}}, "FAIL: $_") foreach (@FAIL); push(@{$result{'details'}}, "CRITICAL: $_") foreach (@CRITICAL); push(@{$result{'details'}}, "WARN: $_") foreach (@WARN); push(@{$result{'details'}}, "DETAILS: $_") foreach (@DETAILS); push(@{$result{'details'}}, "INFO: $_") foreach (@INFO); push(@{$result{'details'}}, "$_") foreach (@DEBUG); push(@RESULTS, \%result); @FAIL = (); @CRITICAL = (); @WARN = (); @INFO = (); @DETAILS = (); @DEBUG = (); %MISSING = (); return; } # End printResults sub printSummary { local $"="\n"; my @output = (); foreach my $r_href (@RESULTS) { my %result = %$r_href; if ($result{'value'} <= 75) { push(@output, "========================="); push(@output, "$result{'name'}"); push(@output, "========================="); push(@output, "@{$result{'details'}}"); } } if (@output) { print colored("\nSummary of checks with warnings or failures:", 'bold underline'); print "\n\n@output\n"; } else { print colored("\nAll checks passed\n\n", 'bold underline'); } return; } 1; package Parsers; use strict; use warnings; use Exporter qw( import ); use Time::Local; eval { require Logger }; import Logger qw( debug ); our $VERSION = v1.1.0.0; BEGIN { our @EXPORT_OK = qw ( parseOneFSVersion parseRules parseCoreCount parseGconfigItem parseGconfigList parseTime ); our %EXPORT_TAGS = ( ALL => \@EXPORT_OK ); } sub parseOneFSVersion { return () unless @_ and $_[0] and $_[0] =~ /^0x([0-9a-f]+)$/i; my $h = $1; my %version = ( 'major' => hex(substr $h, -16, 2), 'minor' => hex(substr $h, -14, 2), 'sub' => hex(substr $h, -12, 2), 'style' => hex(substr $h, -10, 1), 'dot' => hex(substr $h, -7, 2), 'build' => hex(substr $h, -5), ); return \%version; } sub parseRules { # Operators: Append -> ++, Insert -> +=, Replace -> =>, Remove Groups -> --, Join -> &= # Options: [] my @rules = (); return @rules unless $_[0]; while ($_[0] =~ /\\?"?(\S+)\s+(\+\+|\+=|=>|--|&=)\s+(\S+)\s*(\[\S+\])?\\?"?/g) { my $rule = {}; $rule->{'left'} = $1; $rule->{'operator'} = $2; $rule->{'right'} = $3; $rule->{'options'} = $4 if $4; push(@rules, $rule); debug("parseRules: Left: $rule->{'left'} Operator: $rule->{'operator'} Right: $rule->{'right'}". ($rule->{'options'}?" Options: $rule->{'options'}":"")."\n"); } return @rules; } ### # parseCoreCount: Parses the PROC line from isi_hw_status and return the number of cores (int). sub parseCoreCount { my ($line) = @_; return 0 unless $line; $_ = $line; s/-HT//; s/Single/1/; s/Dual/2/; s/Quad/4/; s/Hexa/6/; s/Octa/8/; my ($proc, $core) = (1, 1); $proc = $1 if /(\d+)-proc/; $core = $1 if /(\d+)-core/; return int($proc * $core); } # End parseCoreCount sub parseGconfigItem { my ($gconfig) = @_; my ( $key, $value ) = (q{}) x 2; if ( !defined $gconfig ) { $gconfig = q{} } debug("parseGconfigItem: Argument: $gconfig"); if ( $gconfig =~ /\s=/xms ) { ( $key, $value ) = split /=/xms, $gconfig, 2; $key =~ s/^(\S+).*$/$1/xms; $value =~ s/^\s*"?(\S.*?)"?$/$1/xms; } debug("parseGconfigItem: Return: [ $key, $value ]"); return $key, $value; } sub parseGconfigList { my ($gconfig) = @_; my ( $key, $value ) = (q{}) x 2; my @values = (); if ( !defined $gconfig ) { $gconfig = q{} } debug("parseGconfigList: Argument: $gconfig"); ( $key, $value ) = parseGconfigItem($gconfig); $value =~ s/^\s*\[\s*|\s*\]\s*$|\\"//gxms; if ( $value !~ /^\[\]$/xms ) { @values = split /,\s/xms, $value } foreach (@values) { s/^"|"$//gxms } debug("parseGconfigList: Return: [ $key, @values ]"); return $key, @values; } sub parseTime { my ($str) = @_; if (!$str) { return } my ($year, $mon, $day, $hour, $min, $sec, $tz); debug("Parsing $str"); # Trim whitespace $str =~ s/^\s*|\s*$//g; # Parse date in yyyy-mm-dd format if ($str =~ /(\d{4})-(\d{2})-(\d{2})/) { ($year, $mon, $day) = ($1, $2, $3); } # Parse time in hh:mm:ss format if ($str =~ /(\d{2}):(\d{2}):(\d{2})/) { ($hour, $min, $sec) = ($1, $2, $3); } # Parse time zone in either Z, +hh:mm, or -hh:mm format if ($str =~ /Z$/) { $tz = 0; } elsif ($str =~ /-(\d{2}):(\d{2})$/) { $tz = $1 * 60 * 60 + $2 * 60; } elsif ($str =~ /\+(\d{2}):(\d{2})$/) { $tz = -1 * $1 * 60 * 60 + $2 * 60; } # Convert the time into an epoch value my $epoch = 0; eval { $epoch = timegm($sec, $min, $hour, $day, $mon - 1, $year) + $tz }; if ($@) { debug($@) } # Return the epoch value debug("Returning $epoch"); return $epoch; } 1; package main; exit IOCA::main();