#!/bin/sh

# There is a race condition in isi_version_checker that effects large clusters
# so to mitigate we back off for 1-10 seconds on a failure and try again. We
# will try this up to 5 times before allowing a failure.

retry=0
while [ $retry -lt 5 ]; do
    retry=$(($retry + 1));

    # Sleep for random time between 1 and 5 seconds
    /bin/sleep `/usr/bin/jot -r 1 1 10`;

    # Using locks and random number approach to create a window of AT-MOST 10
    # isi_upgrade_checker instances active at any point inside the cluster. We
    # do not want to serialize it as this script takes longer than 60 seconds
    # to finish and we'll add several minutes to the upgrade on large clusters
    # by serializing this hook.
    /usr/bin/lockf \
        /ifs/.ifsvar/tmp/isi_upgrade_checker.`/usr/bin/jot -r 1 1 10`.lock \
        /usr/bin/isi_version_checker --upgrade --on_cluster \
        --installtar=/ifs/.ifsvar/upgrade/install.tar.gz && break;
done

exit $?
