#!/bin/sh

# Copyright (c) 2012 ken.naruo
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.


LANG=C;export LANG
SCRIPT_DIR=`dirname $0`
JOBNET=`echo ${SCRIPT_DIR} | awk -F [/] '{field = $NF } END {print field }'`
PROJECT=`echo ${SCRIPT_DIR} | awk -F [/] '{field = $(NF-1) } END {print field}'`
JOB_EXEC_USER=ejobmgr
SUDO_COMMAND="" #null

# .root load
. ${SCRIPT_DIR}/../../../.root

JOBNET_HOME=${SCRIPT_DIR}
JOBNET_LOG=${SCRIPT_DIR}/log/jobnet_log_`date +%Y%m%d`
STATUS_QUE=${ROOT}/que/status/${PROJECT}_${JOBNET}
ERROR_QUE=${ROOT}/que/error/${JOBNET}_`date +%s`
TMP_QUE=${ROOT}/que/tmp/${PROJECT}::${JOBNET}
SPARE_FLAG="${SCRIPT_DIR}/spare"


## jobnet prof load
grep -v SCHEDULE ${SCRIPT_DIR}/jobnet.prof > ${SCRIPT_DIR}/.jobnet_tmp
chmod 766 ${SCRIPT_DIR}/.jobnet_tmp
. ${SCRIPT_DIR}/.jobnet_tmp

STATUS_QUE_INPUT_MESSAGE=${ROOT}/que/status/${PROJECT}_${JOBNET}_IN_MESSAGE
STATUS_QUE_OUTPUT_MESSAGE=${ROOT}/que/status/${PROJECT}_${JOBNET}_OUT_MESSAGE

## spare check
if [ -f "${SPARE_FLAG}" ];then
  SERVER=`cat ${SPARE_FLAG}`
  ping -c 3 ${SERVER}
  if [ $? = 0 ];then
    exit 0
  else
    sleep 60
    ping -c 3 ${SERVER}
    if [ $? = 0 ];then
      exit 0
    fi
  fi
fi


#jobnet start
touch ${STATUS_QUE}


if [ "${JOB_EXEC_USER}" != ejobmgr ];then

  SUDO_COMMAND="sudo su ${JOB_EXEC_USER} -c"

fi



## mesage check
if [ "${INPUT_MESSAGE}" ];then
 for MESSAGE in ${INPUT_MESSAGE}
 do
   if [ ! -f "${ROOT}/que/message/${MESSAGE}" ]; then
     echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} input message ${MESSAGE} not set" > ${STATUS_QUE_INPUT_MESSAGE}
     exit 1
   fi
 done

 echo "`date +%Y/%m/%d` `date +%H:%M:%S` input message ${INPUT_MESSAGE} exist" >> ${JOBNET_LOG}
 echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} input message ${INPUT_MESSAGE} exist" > ${STATUS_QUE_INPUT_MESSAGE}

fi

##Remove input message
if [ "${INPUT_MESSAGE}" ];then
  for MESSAGE in ${INPUT_MESSAGE}
  do
   rm -f ${ROOT}/que/message/${MESSAGE}
  done
fi

## job exexute

touch ${TMP_QUE}

for JOB in $JOB_ORDER
do
 RECOVERY_COMMAND=""
 ERROR_FLAG=""
 EXCLUSIVE_RESOURCE=""

 #job load
 . ${SCRIPT_DIR}/${JOB}
 JOB_LOG=${SCRIPT_DIR}/log/${JOB}_log_`date +%Y%m%d`

 #EXCLUSIVE resource check
 count=0
 sleep_time=10

 if [ "${EXCLUSIVE_RESOURCE}" ];then
  while [ -f "${ROOT}/que/exclusive_resource/${EXCLUSIVE_RESOURCE}" ]
  do
    WARN_FLAG=EXCLUSIVE_RESOURCE_LOCKED_BY_OTHER_JOB
    echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} WARNING ${WARN_FLAG}" >> ${JOBNET_LOG}
    echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${WARN_FLAG}" > ${STATUS_QUE}
    echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${WARN_FLAG}" >> ${ERROR_QUE}

    if [ "${MAIL_ADDR}" ];then
     TIME=`date +%s`
     MAIL_QUE=${ROOT}/que/mail/${PROJECT}_${JOBNET}_${TIME}
     echo "MAIL_ADDR:${MAIL_ADDR}" > ${MAIL_QUE}
     echo "SUBJECT:JOBNET:${JOBNET} not start" >> ${MAIL_QUE}
     echo "CONTENTS:`date +%Y/%m/%d` `date +%H:%M:%S` ${EXCLUSIVE_RESOURCE} resource locked by other job" >> ${MAIL_QUE}
    fi

    sleep ${sleep_time}
    count=`echo ${count}+1 | bc`
    sleep_time=`echo ${sleep_time}*2 | bc`

    if [ "${count}" = 20 ];then
      ERROR_FLAG=EXCLUSIVE_RESOURCE_CHECK_OVER_RETRY_COUNT
      echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} failed ${ERROR_FLAG}" >> ${JOBNET_LOG}
      echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG}" > ${STATUS_QUE}
      echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG}" >> ${ERROR_QUE}

      if [ "${MAIL_ADDR}" ];then
        TIME=`date +%s`
        MAIL_QUE=${ROOT}/que/mail/${PROJECT}_${JOBNET}_${JOB}_${TIME}
        echo "MAIL_ADDR:${MAIL_ADDR}" > ${MAIL_QUE}
        echo "SUBJECT:JOBNET:${JOBNET} retry count over" >> ${MAIL_QUE}
        echo "CONTENTS:`date +%Y/%m/%d` `date +%H:%M:%S` ${EXCLUSIVE_RESOURCE} resource locked by other job" >> ${MAIL_QUE}
      fi

      rm -f ${STATUS_QUE}
      rm -f ${TMP_QUE}
      exit 1

    fi

  done

 fi


 #JOB start
 if [ "${EXCLUSIVE_RESOURCE}" ];then
   echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} locked ${EXCLUSIVE_RESOURCE} resource" > ${ROOT}/que/exclusive_resource/${EXCLUSIVE_RESOURCE}
 fi

 echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} START "   >> ${JOBNET_LOG}
 echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} START " > ${STATUS_QUE}
 echo "`date +%Y/%m/%d` `date +%H:%M:%S` EXEC ${EXECUTE_COMMAND} " >> ${JOB_LOG}

 if [ "${JOB_EXEC_USER}" = ejobmgr ];then
   ${EXECUTE_COMMAND}  >> ${JOB_LOG} 2>&1
 else
   ${SUDO_COMMAND} ${EXECUTE_COMMAND}  >> ${JOB_LOG} 2>&1
 fi

 EXECUTE_COMMAND_CHECK=$?

 if [ "${EXCLUSIVE_RESOURCE}" ];then
   rm -f ${ROOT}/que/exclusive_resource/${EXCLUSIVE_RESOURCE}
 fi

 if [ "${EXECUTE_COMMAND_CHECK}" != 0 ];then
   ERROR_FLAG=EXECUTE_COMMAND_FAILED
   echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} ERROR ${ERROR_FLAG} " >> ${JOBNET_LOG}
   echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG}" > ${STATUS_QUE}
   echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG} " >> ${ERROR_QUE}

   if [ "${MAIL_ADDR}" ];then
     TIME=`date +%s`
     MAIL_QUE=${ROOT}/que/mail/${PROJECT}_${JOBNET}_${JOB}_${TIME}
     echo "MAIL_ADDR:${MAIL_ADDR}" > ${MAIL_QUE}
     echo "SUBJECT:JOBNET:${JOBNET} failed" >> ${MAIL_QUE}
     echo "CONTENTS:`date +%Y/%m/%d` `date +%H:%M:%S` ERROR ${PROJECT} ${JOBNET} ${ERROR_FLAG}" >> ${MAIL_QUE}
   fi

   if [ "${RECOVERY_COMMAND}" ];then
     echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} RECOVERY START" > ${STATUS_QUE}
     echo "`date +%Y/%m/%d` `date +%H:%M:%S` EXEC ${RECOVERY_COMMAND} " >> ${JOB_LOG}

     if [ "${JOB_EXEC_USER}" = ejobmgr ];then
       ${RECOVERY_COMMAND} >> ${JOB_LOG} 2>&1
     else
       ${SUDO_COMMAND} ${RECOVERY_COMMAND} >> ${JOB_LOG} 2>&1
     fi

     RECOVERY_COMMAND_CHECK=$? >> ${JOB_LOG} 2>&1

     if [ "${RECOVERY_COMMAND_CHECK}" != 0 ];then
       ERROR_FLAG=RECOVERY_FAILED
       echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG} " > ${STATUS_QUE}
       echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} ${ERROR_FLAG} " >> ${JOBNET_LOG}
       echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG} " >> ${ERROR_QUE}

       if [ "${MAIL_ADDR}" ];then
         TIME=`date +%s`
         MAIL_QUE=${ROOT}/que/mail/${PROJECT}_${JOBNET}_${JOB}_RECOVERY_${TIME}
         echo "MAIL_ADDR:${MAIL_ADDR}" > ${MAIL_QUE}
         echo "SUBJECT:JOBNET:${JOBNET} recovery failed" >> ${MAIL_QUE}
         echo "CONTENTS:`date +%Y/%m/%d` `date +%H:%M:%S` ERROR ${PROJECT} ${JOBNET} ${JOB} ${ERROR_FLAG}" >> ${MAIL_QUE}
       fi

       rm -f ${TMP_QUE}
       exit 1

     else
       echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} RECOVERY COMPLETE " > ${STATUS_QUE}
       echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} RECOVERY COMPLETE" >> ${JOBNET_LOG}

       if [ "${MAIL_ADDR}" ];then
         TIME=`date +%s`
         MAIL_QUE=${ROOT}/que/mail/${PROJECT}_${JOBNET}_${JOB}_RECOVERY_${TIME}
         echo "MAIL_ADDR:${MAIL_ADDR}" > ${MAIL_QUE}
         echo "SUBJECT:JOBNET:${JOBNET} recovery complete" >> ${MAIL_QUE}
         echo "CONTENTS:`date +%Y/%m/%d` `date +%H:%M:%S` INFO ${PROJECT} ${JOBNET} ${JOB} recovery complete" >> ${MAIL_QUE}

       fi

       rm -f ${TMP_QUE}
       exit 1
       
     fi
     
   else
   
     rm -f ${TMP_QUE}
     exit 1   
     
   fi
   
 else

   #Job End
   echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${JOB} END ${EXECUTE_HOST_ADDR}"   >> ${JOBNET_LOG}
   echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ${JOB} END ${EXECUTE_HOST_ADDR}" > ${STATUS_QUE}

 fi

done


#Create output message
if [ "${OUTPUT_MESSAGE}" ];then

  touch ${ROOT}/que/message/${OUTPUT_MESSAGE}
  echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} output message ${OUTPUT_MESSAGE} generate" > ${STATUS_QUE_OUTPUT_MESSAGE}
  echo "`date +%Y/%m/%d` `date +%H:%M:%S` output message ${OUTPUT_MESSAGE} generate" >> ${JOBNET_LOG}

fi

## Jobnet end
echo "`date +%Y/%m/%d` `date +%H:%M:%S` ${PROJECT} ${JOBNET} ALL JOB NORMAL END " > ${STATUS_QUE}
rm -f ${TMP_QUE}
