#!/usr/bin/bash
# NDU Precheck for BURT 1078977 IGMP refcount problem

# This log is saved on a single node of cluster:
logdir="/mroot/etc/log/mlog"
logf="$logdir/andu-igmp-check-fix.log"
touch $logf; echo >> $logf; echo >> $logf; date >> $logf
echo Starting NDU Precheck for IGMP refcount problem in KB ID SU305 >> $logf

# Allow forced bypass of this NDU precheck script as needed by customer:
bypass_sig="ForcePrecheckPass"
bypass_log=`grep $bypass_sig $logf | grep -o $bypass_sig | tail -n 1`
if [ "$bypass_log" = "$bypass_sig" ]; then
    echo Bypass NDU precheck for KB ID SU305 and return success. >> $logf
    exit 0
fi

# Note:
# Filter out the nodes that are not running 9.2 so that this precheck is skipped when ANDU/NDU "from" release is not 9.2.
# This script will automatically phase out as more systems upgrade away from 9.2.
# Releases of R9.2P3 and later will not execute this script because IGMP counter is always zero.
ver_filter="9.2"

# Initialize:
declare -a node_array
declare -a skip_array
node_array=("local")
node_cnt=${#node_array[@]}
fixed_flag_file=$logf
fixed_signature="KB1074346_IGMP_REFCNT_CHECK_AND_FIX_COMPLETED"
max_repair="2400"
sleep_seconds=20
need_manual_workaround_in_KB="NO"

error_msg=""
show_error_and_abort () {
    echo >> $logf; date >> $logf
    echo Error: Precheck failed by systemshell command error: $error_msg, return precheck failure >> $logf
    # Leave a signature to indicate check and fix is applied:
    echo Leaving a signature to prevent future precheck as manual workaround will be done >> $logf
    for (( i=0; i<${node_cnt}; i++ ));
    do
	node=${node_array[$i]}
	sudo /sbin/ngsh -c "set d; systemshell -node $node -command echo $fixed_signature >> $fixed_flag_file" > /dev/null
        echo fixed signature written to node $node $fixed_flag_file >> $logf
    done
    exit 1
}

# Walk the nodes and identify which one to filter out:
echo >> $logf; date >> $logf; echo Step 1: Filtering nodes >> $logf
for (( i=0; i<${node_cnt}; i++ ));
do
    node=${node_array[$i]}
    # Filter out nodes running inapplicable version:
    version=`sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command sudo cat /VERSION" | grep "$ver_filter" | grep -o "$ver_filter"`
    echo >> $logf; echo Step 1-1: version of node $node = $version >> $logf
    if [ "$version" != "" ] && [ "$version" != "$ver_filter" ]; then
       error_msg="cat VERSION failed on $node"
       show_error_and_abort
    fi
    echo processing node[$i] $node version = $version >> $logf
    if [ "$version" != "$ver_filter" ]; then
	echo Step 1-1: skipping node $node as version is $version >> $logf
	skip_array[$i]="SKIP"
	continue
    fi
    # Filter out nodes that are already addressed:
    already_fixed=`sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command grep $fixed_signature $fixed_flag_file" | strings | grep -o $fixed_signature | tail -n 1`
    if [ "$already_fixed" = "$fixed_signature" ]; then
	echo Step 1-2: skipping node $node as signature $already_fixed exists >> $logf
	skip_array[$i]="SKIP"
	continue
    fi
    # Filter out nodes that show zero IGMP send count:
    igmp_count=`sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command sudo jexec Cluster netstat -s | grep 'membership reports sent'" | strings | grep membership | xargs | cut -d' ' -f1`
    if [ "$igmp_count" == "" ]; then
       error_msg="netstat IGMP counter failed on $node"
       show_error_and_abort
    fi
    if [ "$igmp_count" = "0" ]; then
	echo Step 1-3: skipping node $node as IGMP send count is $igmp_count >> $logf
	skip_array[$i]="SKIP"
	continue
    fi
    skip_array[$i]="FIX_IT"
    ((total_nodes_to_fix++))
    if [[ "$igmp_count" -gt "$max_repair" ]]; then
	echo Step 1-4: node $node has igmp_count = $igmp_count, need manual workaround >> $logf
	need_manual_workaround_in_KB="YES"
    fi
done
echo Step 1-5: total $total_nodes_to_fix nodes will be repaired >> $logf

# Placing function here to make it easier to read. It is called right below.
execute_one_iteration () {
    for (( i=0; i<${node_cnt}; i++ ));
    do
	node=${node_array[$i]}
	skip=${skip_array[$i]}
	# Step 2-1. skip if fix is not required:
	if [ "$skip" = "SKIP" ]; then
	    echo Step 2-1: skipping starting iftest on node $node because skip = $skip >> $logf
	    continue
	fi
	# Step 2-2. Execute iftest in the background to repair each node as needed:
	sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command sudo jexec Cluster iftest multicast 0 & " > /dev/null
	pid_iftest=`sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command ps a" | strings | grep iftest | grep -v grep | cut -d' ' -f1`
	echo Step 2-1: on node $node, started iftest process: pid = $pid_iftest >> $logf
	((total_fixed++))
    done
    # Step 2-3. Wait for enough time for iftest to finish creating all the sockets:
    sleep $sleep_seconds
    for (( i=0; i<${node_cnt}; i++ ));
    do
	node=${node_array[$i]}
	skip=${skip_array[$i]}
	# Step 2-4. skip if fix is not required:
	if [ "$skip" = "SKIP" ]; then
	    echo Step 2-4: skipping stopping iftest on node $node because skip = $skip >> $logf
	    continue
	fi
	# Step 2-5. Stop iftest on each node:
	sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command sudo pkill iftest" > /dev/null
	pid_iftest=`sudo /sbin/ngsh -c "set d; rows 0; systemshell -node $node -command ps a" | strings | grep iftest | grep -v grep | cut -d' ' -f1`
	echo Step 2-5: iftest stopped on node $node, checking pid of iftest = $pid_iftest >> $logf
    done
}

# Walk the nodes and iterate repair operations:
if [[ total_nodes_to_fix -eq 0 ]]; then
    echo >> $logf; date >> $logf
    echo Step 2: No iteration as no nodes require fix >> $logf
else
    for (( x=1; x<4; x++ ));
    do
	echo >> $logf; date >> $logf
	echo Step 2: Running iftest, iteration count = $x >> $logf
	execute_one_iteration
	sleep 1
    done
fi

# Leave a flag file to indicate fix is applied:
echo >> $logf; date >> $logf
echo Step 3: Leaving a flag file to indicate fix is done >> $logf
for (( i=0; i<${node_cnt}; i++ ));
do
    node=${node_array[$i]}
    # Note: we need to mark all the nodes regardless of filtering decision, otherwise, next upgrade will get stuck
    sudo /sbin/ngsh -c "set d; systemshell -node $node -command echo $fixed_signature >> $fixed_flag_file" > /dev/null
    echo Step 3-1: fixed signature written to node $node $fixed_flag_file >> $logf
done

# Making final decision and return pass or failure:
echo >> $logf; date >> $logf
echo Step 4: Making final decision - pass or fail >> $logf
if [ "$need_manual_workaround_in_KB" = "YES" ]; then
    echo Step 4-1: Precheck faild by need_manual_workaround_in_KB = $need_manual_workaround_in_KB, suggest manual workaround >> $logf
    exit 1
else
    echo Step 4-2: Precheck passed >> $logf
    exit 0
fi
