#!/bin/ksh
#-----------------------------------------------------------------------------#
#                                                                             #
#     STEP_DRIVER                                                             #
# Description:                                                                #
#  Called by a shell script which has defined steps, step descriptions and    #
#  functions in the following format:                                         #
#                                                                             #
#  FIRST_STEP=10                                                              #
#  STEP_NAME[10]="step_function_name"                                         #
#  STEP_DESC[10]="step one line description"                                  #
#  LAST_STEP=10                                                               #
#  step_function_name()                                                       #
#  {                                                                          #
#  }                                                                          #
#                                                                             #
# Invocation:                                                                 #
# . step_driver                                                               #
# as last line in shell script described above.                               #
#                                                                             #
#-----------------------------------------------------------------------------#
# Modification history:                                                       #
# Date     Programmer            Description                                  #
# 94/06/20 Michael Greenberg     inital version                               #
# 95/04/27    "      "           added check for STDERR_TEXT and trap SIGWINCH#
# 96/03/24    "      "           handle run via remote shell and parallel task#
# 97/02/27    "      "           lock out two tasks running with same disk/dat#
#                                                                             #
#-----------------------------------------------------------------------------#


#---------------------------------------------------------------------#
#                        list_steps                                   #
# list step names and descriptions for requested steps                #
#---------------------------------------------------------------------#
list_steps()
{
	echo "Step listing for program ${SHELL_NAME}:"
	echo
	CURRENT_STEP=$FROM_STEP

	while [ "${CURRENT_STEP}" -le "${TO_STEP}" ]
	do
		echo "STEP: ${CURRENT_STEP}"
		echo "   STEP_NAME: ${STEP_NAME[${CURRENT_STEP}]}"
		echo "   DESCRIPTION: ${STEP_DESC[${CURRENT_STEP}]}"
		sed -n "/${STEP_NAME[${CURRENT_STEP}]}()/,/^}$/p" ${PROGNAME} | \
			sed -n -e "s/\\\\//" \
				-e "/SQR_DIR/s/.*SQR_DIR}\//     sqr /p" \
				-e "/}\/cosort /s/.*/     cosort/p" \
				-e "/[ 	]np /s/.*/     np/p" \
				-e "/[ 	]lp /s/.*/     lp/p"
#
#	find next step
#
		((CURRENT_STEP += 1))
		while [ "${STEP_NAME[${CURRENT_STEP}]}" = "" -a ${CURRENT_STEP} -le ${TO_STEP} ]
		do
			((CURRENT_STEP += 1))
		done
	done
}
#---------------------------------------------------------------------#
#                        run_concurrent                               #
# run functions passed as parameters concurrently                     #
#---------------------------------------------------------------------#
run_concurrent()
{

# check that a recursive call to run_concurrent has not been made
	if [ "${in_concurrent}" = "true" ]
	then
		echo "Forbidding recursive call to run_concurrent and ABORTING!" | tee -a ${LOG_FILE}
		return 1
	else
		in_concurrent="true"
	fi

	ret=0
	params="$*"

	echo "Commencing concurrent execution of functions" | tee -a ${LOG_FILE}
	SAVE_LOG_FILE=${LOG_FILE}
	SAVE_STDERR_TEXT=${STDERR_TEXT}
	TRUNC_LOG_NAME=`echo ${SAVE_LOG_FILE} | sed "s/\.log/.${CURRENT_STEP}/"`
	set -- ${params}
	CONCURRENT_ID=1
	while [ $# -gt 0 ]
	do
		if [ ! -f ${CONCURRENT_COMPLETED}.${CONCURRENT_ID} ]
		then
			THIS_FUNCTION=$1
			STDERR_TEXT=${SAVE_STDERR_TEXT}.${CONCURRENT_ID}
			LOG_FILE=${TRUNC_LOG_NAME}.${THIS_FUNCTION}.${CONCURRENT_ID}.log
			ALL_LOGS="${ALL_LOGS} ${LOG_FILE}"
			rm -f ${LOG_FILE} ${LOG_FILE}.result
			echo "LOG_FILE=${LOG_FILE}"
			echo "${CONCURRENT_ID}\t${THIS_FUNCTION}" >> ${SAVE_LOG_FILE}
			echo "start function ${THIS_FUNCTION} with CONCURRENT_ID=${CONCURRENT_ID} on `date`" > ${LOG_FILE}
			(
			${THIS_FUNCTION}; \
			echo $? > ${LOG_FILE}.result;
			echo "finish function ${THIS_FUNCTION} `date`" >> ${LOG_FILE}
			) &
		fi
		((CONCURRENT_ID += 1))
		shift
	done

# restore log file
	LOG_FILE=${SAVE_LOG_FILE}

# wait for all task to complete
	wait

	echo "completed last concurrent function on `date`" >> ${LOG_FILE}

	set -- ${params}
	CONCURRENT_ID=1
	while [ $# -gt 0 ]
	do
		if [ ! -f ${CONCURRENT_COMPLETED}.${CONCURRENT_ID} ]
		then
			THIS_FUNCTION=$1
			echo >> ${SAVE_LOG_FILE}
			echo "Log file from concurrent function: ${THIS_FUNCTION}" >> ${SAVE_LOG_FILE}
			LOG_FILE=${TRUNC_LOG_NAME}.${THIS_FUNCTION}.${CONCURRENT_ID}.log
			if [ -f ${LOG_FILE} ]
			then
				cat ${LOG_FILE} >> ${SAVE_LOG_FILE}
				if [ $? -eq 0 ]
				then
					rm -f ${LOG_FILE}
				fi
			fi
			if [ ! -f ${LOG_FILE}.result -o ! -s ${LOG_FILE}.result ]
			then
				echo "concurrent step: ${THIS_FUNCTION} missing result file assuming failure." | tee -a ${SAVE_LOG_FILE}
				ret=1
			else
				result=`cat ${LOG_FILE}.result`
				if [ "$result" != "0" ]
				then
					ret=1
					echo "concurrent step: ${THIS_FUNCTION} failed.  return_code = $ret" | tee -a ${SAVE_LOG_FILE}
				else
					touch ${CONCURRENT_COMPLETED}.${CONCURRENT_ID}
				fi
				rm -f ${LOG_FILE}.result
				rm -f ${SAVE_STDERR_TEXT}.${CONCURRENT_ID}
			fi
		fi
		((CONCURRENT_ID += 1))
		shift
		done
	LOG_FILE=${SAVE_LOG_FILE}
	STDERR_TEXT=${SAVE_STDERR_TEXT}

	in_concurrent="false"

	return ${ret}
}
#---------------------------------------------------------------------#
#                        Check return code                            #
# Checks the return status of the last run program and aborts         #
# execution if the status was failure.                                # 
# It also checks to see if file ${STDERR_TEXT} has a contents         #
#---------------------------------------------------------------------#
check_return_code()
{
	return_code=$?
	if [ ${return_code} -ne 0 -o -s "${STDERR_TEXT}" ]
	then
		if [ -s "${STDERR_TEXT}" ]
		then
			echo "Error message written to stderr:" | tee -a ${LOG_FILE}
			cat "${STDERR_TEXT}" | tee -a ${LOG_FILE}
		fi
		if [ "${in_concurrent}" = "true" ]
		then
			echo ${return_code} > ${LOG_FILE}.result;
			echo "concurrent function ${THIS_FUNCTION} FAILED on `date`."
			echo "Processing of other functions continues ..."
		else
			echo Step ${CURRENT_STEP} ${STEP_NAME[${CURRENT_STEP}]}   \
				FAILED on `date` with exit code ${return_code} | \
				tee -a ${LOG_FILE}
		fi
		do_exit 1
	fi
}

#---------------------------------------------------------------------#
#                   operator pressed 'break'                          #
# - code may be added to clean up temporary files                     #
#---------------------------------------------------------------------#
graceful_exit()
{
# ignore further interrupts
	trap "" 2

# clean up lock file
	if [ -n "${LOCK_FILE}" ]
	then
		rm -f ${LOCK_FILE}
	fi

	echo "${SHELL_NAME} received an INTERRUPT on `date`" | tee -a ${LOG_FILE}
	if [ "${in_concurrent}" = "true" ]
	then
		echo "amalgamating log files from concurrent tasks"
		for file in `echo "${ALL_LOGS}"`
		do
			echo "Contents of $file" >> ${LOG_FILE}
			if [ -f ${file} ]
			then
				cat ${file} >> ${LOG_FILE}
				if [ $? -eq 0 ]
				then
					rm -f ${file}
				fi
			fi
		done
	fi
	echo "${SHELL_NAME} killing all child processes and committing SUICIDE!" >> ${LOG_FILE}

# kill all processes in this process group
	id | grep -s root
	if [ $? -ne 0 ]
	then
		kill 0
	fi

# the following lines should be unreachable
	echo "Exiting..."
	do_exit 1
}

#---------------------------------------------------------------------#
#                        Step Mark Completed                          #
# save the last successfully run step in a sequence file              #
#---------------------------------------------------------------------#
step_mark_completed()
{
	if [ -z "${LAST_STEP_COMPLETED}" ]
	then
		echo "step_mark_completed: environment variable LAST_STEP_COMPLETE is not set."
		do_exit 1
	fi

# save the current step's number in the sequence file
	echo ${CURRENT_STEP} > ${LAST_STEP_COMPLETED}
	if [ $? -ne 0 ]
	then
		echo "step_mark_completed failed to write to sequence file: ${LAST_STEP_COMPLETED}."
		do_exit 1
	fi
}

#---------------------------------------------------------------------#
#                        Step Verify Sequence                         #
#    This program is used by shell scripts that are split into steps. #
#    It verifies that the previous step completed successfully.       #
#---------------------------------------------------------------------#
step_verify_sequence()
{
	missed_steps=""

	if [ -z "${LAST_STEP_COMPLETED}" ]
	then
		echo "step_verify_sequence: environment variable LAST_STEP_COMPLETE is not set."
		do_exit 1
	fi

	if [ -s "${LAST_STEP_COMPLETED}" ]
	then
# read the last successfully completed step from the sequence file
		last_step=`cat ${LAST_STEP_COMPLETED}`
		((i = ${CURRENT_STEP} - 1))
	
# check for existence of steps between the current and the last run
		while [ ${i} -ge 0 -a ${i} -gt ${last_step} ]
		do
			if [ "${STEP_NAME[${i}]}" != "" ]
			then
				missed_steps="${missed_steps} ${i}"
				echo "${SHELL_NAME}: WARNING step ${i} \"${STEP_NAME[${i}]}\" has not completed successfully." | tee -a ${LOG_FILE}
			fi
			((i -= 1))
		done
	else
# check for existence of steps between the first step and the current step
		i=${FIRST_STEP}
		while [ ${i} -lt ${CURRENT_STEP} ]
		do
			if [ "${STEP_NAME[${i}]}" != "" ]
			then
				missed_steps="${missed_steps} ${i}"
				echo "${SHELL_NAME}: Warning step ${i} \"${STEP_NAME[${i}]}\" has not completed successfully." | tee -a ${LOG_FILE}
			fi
			((i += 1))
		done
	fi
	
# prompt the operator to continue if missing steps existed
	if [ "${missed_steps}" != "" ]
	then
		if [ -n "${TTY}" ]
		then
			echo "Are you sure that you want to continue running the program \"${SHELL_NAME}\"?"
			echo "(y, n default is n): \c"
			read ans
			if [ "${ans}" = "y" -o "${ans}" = "Y" ]
			then
				echo >> ${LOG_FILE}
				echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> ${LOG_FILE}
				echo "WARNING: operator running step ${CURRENT_STEP} despite missing step(s) ${missed_steps} \non `date`" >> ${LOG_FILE}
				echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> ${LOG_FILE}
				echo >> ${LOG_FILE}
			else
				echo "${SHELL_NAME} execution CANCELLED by operator on `date`" >> ${LOG_FILE}
				echo "Exiting..."
				do_exit 1
			fi
		else
#			echo >> ${LOG_FILE}
#			echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> ${LOG_FILE}
#			echo "WARNING: operator running step ${CURRENT_STEP} despite missing step(s) ${missed_steps} \non `date`" >> ${LOG_FILE}
#			echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> ${LOG_FILE}
#			echo >> ${LOG_FILE}
			echo "Unable to continue due to missing steps" | tee -a ${LOG_FILE}
			echo "${SHELL_NAME} execution ABORTED on `date`" >> ${LOG_FILE}
			do_exit 1
		fi
	fi
}

do_exit()
{
	exit_code=$1

	if [ -z "${exit_code}" ]
	then
		exit_code=0
	fi

	if [ "${in_concurrent}" != "true" ]
	then
# clean up lock file
		if [ -n "${LOCK_FILE}" ]
		then
			rm -f ${LOCK_FILE}
		fi
	fi

	if [ "${exit_code}" -gt 0 -a "${in_concurrent}" != "true" ]
	then
		echo "Additional information may be found in: ${LOG_FILE}"
		echo "${SHELL_NAME} execution ABORTED."
	fi

	exit ${exit_code}
}

#---------------------------------------------------------------------#
#   Program starts here
#---------------------------------------------------------------------#

LOG_DIR=/tmp
YYYYMMDD=`date +"%Y%m%d"`
trap "graceful_exit" 2

# prevent window size change causing problems (bug in HP-UX 9)
trap "" SIGWINCH

PROGNAME=$0
SHELL_NAME=`basename $0`

# set FROM_STEP and TO_STEP if they were not specified
if [ -z "${FROM_STEP}" ]
then
	FROM_STEP=${FIRST_STEP}
fi

if [ -z "${TO_STEP}" ]
then
	TO_STEP=${LAST_STEP}
fi

if [ ${FROM_STEP} -gt ${TO_STEP} ]
then
	echo "${SHELL_NAME}: FROM_STEP ${FROM_STEP} is greater than TO_STEP ${TO_STEP}" 
	do_exit 1
fi

if [ -n "${STEP_LIST_ONLY}" ]
then
	list_steps
	do_exit 0
fi

# set working directory if DISK environment variable provided
if [ -n "${DISK}" ]
then
	cd ${DISK}
	if [ $? -ne 0 ]
	then
		echo "${SHELL_NAME}: unable to change directories to ${DISK}"
		do_exit 1
	else
		echo "${SHELL_NAME}: working directory set to ${DISK}"
	fi
fi

# check for existence of same program with same parameters
LOCK_PREFIX=${LOG_DIR}/TS_${SHELL_NAME}_${YYYYMMDD}.lock
LOCK_FILE=${LOCK_PREFIX}.$$
touch ${LOCK_FILE}
if [ $? -ne 0 ]
then
	echo "${SHELL_NAME}: cannot create lock file: ${LOCK_FILE}"
	do_exit 1
fi
for lock in `ls ${LOCK_PREFIX}*`
do
	if [ "${lock}" != "${LOCK_FILE}" ]
	then
# lock file with same DISK and YYYYMMDD exist
		pid=`echo ${lock} | sed "s/.*lock\.//"`
		kill -0 ${pid} >/dev/null 2>&1
		if [ $? -eq 0 ]
		then
			echo "${SHELL_NAME}: duplicate shell:"
			echo "LOG_DIR=${LOG_DIR} YYYYMMDD=${YYYYMMDD} pid: ${pid}"
			do_exit 1
		else
# clean up the old file
			rm -f ${lock}
		fi
	fi
done

# create log file
LOG_FILE=${LOG_DIR}/TS_${SHELL_NAME}_${YYYYMMDD}.log
touch ${LOG_FILE}
if [ $? -ne 0 ]
then
	echo "${SHELL_NAME}: cannot create sequence file: ${LOG_FILE}"
	do_exit 1
fi

# create sequence log file
LAST_STEP_COMPLETED=${LOG_DIR}/TS_${SHELL_NAME}_${YYYYMMDD}.seq
touch ${LAST_STEP_COMPLETED}
if [ $? -ne 0 ]
then
	echo "${SHELL_NAME}: cannot create log file: ${LAST_STEP_COMPLETED}"
	do_exit 1
fi

# create STDERR_TEXT file
STDERR_TEXT=`mktemp`

#
# job banner
#
if [ ${FROM_STEP} -eq ${FIRST_STEP} ]
then
	echo "- START -" >>${LOG_FILE}
fi

# check if un via interactive shell
TTY=`tty`
if [ $? -ne 0 ]
then
	unset TTY
fi

# log that the job was (re-)started
(
echo "START: ${COMMAND_LINE}"
echo
echo "computer: `uname -n` process: $$ tty: ${TTY:-remote}"
echo
echo "------- environment variables -------"
env | sort
echo "------- environment variables -------";echo) >> ${LOG_FILE}

echo "${SHELL_NAME} STARTED on `date`" | tee -a ${LOG_FILE}
echo "LOG_FILE=${LOG_FILE}"
echo "DISK=${DISK}  YYYYMMDD=${YYYYMMDD} FROM_STEP=${FROM_STEP} TO_STEP=${TO_STEP}" | \
	tee -a ${LOG_FILE}
#

if [ "${FROM_STEP}" = "${TO_STEP}" -a -z "${STEP_NAME[${FROM_STEP}]}" ]
then
	echo "Step ${FROM_STEP} does not exist." | tee -a ${LOG_FILE}
	false
	check_return_code
fi

CURRENT_STEP=$FROM_STEP

while [ "${CURRENT_STEP}" -le "${TO_STEP}" ]
do
#	verify that previous steps have been run before proceeding
	step_verify_sequence

#	set up to rerun concurrent functions if not starting at concurrent
	CONCURRENT_COMPLETED=${LAST_STEP_COMPLETED}.${CURRENT_STEP}
	if [ ${CURRENT_STEP} -gt ${FROM_STEP} ]
	then
		rm -f ${CONCURRENT_COMPLETED}*
	fi
#
#	step banner
#
	echo >> ${LOG_FILE}
	echo "STEP ${CURRENT_STEP}" >> ${LOG_FILE}
#
#
	echo "DESCRIPTION:" ${STEP_DESC[${CURRENT_STEP}]} >> ${LOG_FILE}
	echo Step ${CURRENT_STEP} ${STEP_NAME[${CURRENT_STEP}]} BEGIN on `date` | \
		tee -a ${LOG_FILE}
	echo >> ${LOG_FILE}

	${STEP_NAME[${CURRENT_STEP}]}

	echo Step ${CURRENT_STEP} ${STEP_NAME[${CURRENT_STEP}]} END on `date` | \
		tee -a ${LOG_FILE}
	echo "--------------------------------------------------" >> ${LOG_FILE}
#
# call script which marks the completed step number in a sequence file
	step_mark_completed
#
#	find next step
#
	(( CURRENT_STEP += 1 ))
	while [ "${STEP_NAME[${CURRENT_STEP}]}" = "" -a ${CURRENT_STEP} -le ${TO_STEP} ]
	do
		(( CURRENT_STEP += 1 ))
	done
done
#

# clean up stderr text file
rm -f ${STDERR_TEXT}

#job ended banner
#
echo "${SHELL_NAME} FINISHED on `date`" | tee -a ${LOG_FILE}
if [ ${TO_STEP} -eq ${LAST_STEP} ]
then
	echo >> ${LOG_FILE}
	echo "- FINISH -" >> ${LOG_FILE}
fi
do_exit 0
