/opt/product/scripts/users/scripts/syscheck run as swadmin
 
# Julian Ward - syscheck
# New version includes error codes
# Script to automate monitoring of CORRESPONDENCE systems
# Can be run at any time, or continuously
# Checks Filenet, Staffware, Lighthouse3 and Batch jobs to 
# provide an overall system status
#
#  To start, logon as swdamin, cd to /opt/product/scripts/users/scripts,
then
#  ./startsyscheck  (this will run syscheck automatically every 5 mins)
#
#  To stop,  logon as swdamin, cd to /opt/product/scripts/users/scripts,
then
#  ./stopsyscheck
#
#  To run as a one off, logon as swadmin cd /opt/product/users/scripts then
#  ./syscheck
#
# Set date / time variables used
HOST=`uname -n`
YEAR=`date +%Y`
MONTH=`date +%m`
DAY=`date +%d`
HOUR=`date +%H`
MINUTE=`date +%M`
LOGFILEDATE=$YEAR$MONTH$DAY
# Set time exclusion variables
# Default check batch jobs set to 1 = do checks
DOCHECK=1
# BS = Start time ie 20 = 8pm
BS=20
# BE = End time ie 9 = 9am
BE=9
LIBRARYCHECK=0
# SET FILE PATHS
LOGFILE="/opt/product/scripts/users/logs/syscheck.log"
LIGHTHOUSESTATFILE="pulsestat.txt"
LIGHTHOUSELOGDIR="/home/lighthouse/"
STAFFWAREWARNLOG="/opt/product/staffware/sw_warn"
ELOGDIR="/fnsw/local/logs/elogs/"
MAILSENT="/opt/product/scripts/users/logs/mailsentP"
MASTERLOGFILE="/opt/product/scripts/users/logs/syscheckmaster.log"
LOGFILESIZELIMIT=100000
PROBFILE="/opt/product/scripts/users/logs/problemmessage"
FILENETSTAT="/opt/product/scripts/users/logs/capacity"$HOST".tmp"
CSMTEMPFILE="/opt/product/scripts/users/logs/csmtemp"
DOCTEMPFILE="opt/product/scripts/users/logs/doctemp"
PPMOITEMPFILE="/opt/product/scripts/users/logs/ppmoitemp"
ELOGTEMPFILE="/opt/product/scripts/users/logs/elogtemp"
MESSAGEFILE="/opt/product/scripts/users/logs/sysmessage"
#
# DEFINE MAIL RECIPIENTS - IMPORTANT!! Please ensure E-mail addresses are
VALID!
# Define morning status mail recipients here with MORNINGMAILTO variable
 
#MORNINGMAILTO="BGRISOperateonline@centrica.com
ISSSOperationsSupport@centrica.com"
MORNINGMAILTO="OBGCE.Support@cognizant.com"
#
# DEFINE ALERT MAIL RECIPIENTS
# Recipients will recieve E-mail IF these is a problem
for m in `cat sysemails`
do
  MAILTO=$m" "$MAILTO
  #MAILTO="SCCIM&FOFStaff@centrica.co.uk JAMCORRES@centrica.co.uk"
done
#
# DEFINE STANDARD STATUS MAIL MESSAGE
message="LIGHTHOUSE system checks successful.\nThere are no availability
issues.\n\nRegards\n\tLighthouse - Correspondence support\n\n\nPlease do
not respond directly to this e-mail.\nFor further contact, or any queries
regarding Lighthouse, please use details
below.\n\nOBGCE.Support@cognizan.com\n\nStandby / Out of hours 07769
548782\nService Now: JAM-OPS-CORRESPOND"
#
echo "LIGHTHOUSE System monitor - Checked `date`"  > $LOGFILE
echo "System monitor e-mail alerts to: "$MAILTO >> $LOGFILE
echo "Morning status e-mail to: "$MORNINGMAILTO >> $LOGFILE
checklogfilesize()
{
FILESIZE=`wc -c $MASTERLOGFILE | awk '{print $1}'`
if [ $FILESIZE -gt $LOGFILESIZELIMIT ]
then
    #Archive
    compress $MASTERLOGFILE
    cp $MASTERLOGFILE".Z" $MASTERLOGFILE$LOGFILEDATE
    rm $MASTERLOGFILE".Z"
    > $MASTERLOGFILE
fi
}
checkfilenetcache()
{
# - DOC_tool, CSM_tool, PPMOI checks
APP=FileNet
SEVERITY="PROBLEM"
ERRORCODE="FileNet cache"
echo "Check FileNet cache" >> $LOGFILE
#Get Info from DOC PPMOI CSM commands and direct to logfile
pagecache=`grep "Page cache locked" $FILENETSTAT | awk '{print $7}'`
physicalspace=`grep "Physical space" $FILENETSTAT | awk '{print $7}'`
blankcartridges=`grep "Unlabelled" $FILENETSTAT | awk '{print $5}'`
pagelimit=22
phylimit=85
cartlimit=1
if [ $blankcartridges -lt $cartlimit ]
then
    echo "PROBLEM $APP Blank cartridges="$blankcartridges", less than
"$cartlimit" limit"  >> $LOGFILE
else
    echo "OK $APP Blank cartridges="$blankcartridges", more than
"$cartlimit" limit" >> $LOGFILE
fi
if [ $physicalspace -gt $phylimit ]
then
    echo "PROBLEM $APP Physical space="$physicalspace"% , greater than
"$phylimit"% limit"  >> $LOGFILE
else
    echo "OK $APP Physical space="$physicalspace"% , less than "$phylimit"%
limit" >> $LOGFILE
fi
if [ $pagecache -gt $pagelimit ]
then
    echo "PROBLEM $APP Page_cache1="$pagecache"% Locked, > greater than
"$pagelimit"% limit"  >> $LOGFILE
else
    echo "OK $APP Page_cache1="$pagecache"% Locked, < less than or
="$pagelimit"% limit" >> $LOGFILE
fi
}
checkstaffwareprocs()
{
#Check number of STAFFWARE swentobj wisrpc swrpc procs running
APP=StaffWare
SEVERITY="Serious PROBLEM"
ERRORCODE="Staffware processes"
echo "Check STAFFWARE processes running" >> $LOGFILE
SWENTOBJ=`ps -efa|grep swentobj| grep -v grep | wc -l`
SWRPC=`ps -efa|grep swrpcsvr| grep -v grep | wc -l`
WISRPC=`ps -efa|grep wisrpc| grep -v grep | wc -l`
if [ $SWENTOBJ -lt 1 ]
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE swentobj procs =" $SWENTOBJ >> $LOGFILE
else
    echo "OK $APP swentobj procs =" $SWENTOBJ >> $LOGFILE
fi
if [ $WISRPC -lt 40 ] 
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE wisrpc procs="$WISRPC >> $LOGFILE
else
    echo "OK $APP wisrpc procs="$WISRPC >> $LOGFILE
fi
if [ $SWRPC -lt 1 ] 
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE swrpc procs="$SWRPC >> $LOGFILE
else
    echo "OK $APP swsrpc procs="$SWRPC >> $LOGFILE
fi
}
checkfilenetprocs()
{
#Check FileNet processes
APP=FileNet
SEVERITY="Serious PROBLEM"
ERRORCODE="FileNet processes"
echo "Check FileNet Processes running" >> $LOGFILE
#FNBESs=`ps -efa|grep BESs | grep -v grep |wc -l`
FNBES=`ps -efa|grep Bes | grep -v grep |wc -l`
FNINXBG=`ps -efa|grep INXbg | grep -v grep |wc -l`
FNINXU=`ps -efa|grep INXu | grep -v grep | wc -l`
FNINXS=`ps -efa|grep INXs | grep -v grep | wc -l`
FNOCOR=`ps -efa|grep OCOR | grep -v grep | wc -l`
#if [ $FNBESs -lt 1 ] 
#then
#    #Problem as there should be at least one running
#    echo "$APP $SEVERITY $ERRORCODE BESs procs="$FNBESs >> $LOGFILE
#else
#    echo "OK $APP BESs procs="$FNBESs >> $LOGFILE
#fi
if [ $FNBES -lt 2 ] 
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE BES procs="$FNBES >> $LOGFILE
else
    echo "OK $APP BES procs="$FNBES >> $LOGFILE
fi      
if [ $FNINXBG -lt 1 ] 
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE INXbg procs="$FNINXBG >> $LOGFILE
else
    echo "OK $APP INXbg procs="$FNINXBG >> $LOGFILE
fi
if [ $FNINXU -lt 1 ] 
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE INXu procs="$FNINXU >> $LOGFILE
else
    echo "OK $APP INXu procs="$FNINXU >> $LOGFILE
fi
if [ $FNINXS -lt 1 ] 
then
    #Problem as there should be at least one running
    echo "$APP $SEVERITY $ERRORCODE INXs procs="$FNINXS >> $LOGFILE
else
    echo "OK $APP INXs procs="$FNINXS >> $LOGFILE
fi
if [ $FNOCOR -lt 4 ]
then
    #Problem as there should be at least xxx running
    echo "$APP $SEVERITY $ERRORCODE OCOR procs="$FNOCOR >> $LOGFILE
else
    echo "OK $APP OCOR procs="$FNOCOR >> $LOGFILE
fi  
# check to see if there are too many being spawned
if [ $FNOCOR -gt 16 ]
then
    #Problem as there should not be more than xxx running
    echo "$APP $SEVERITY $ERRORCODE OCOR procs="$FNOCOR >> $LOGFILE
else
    echo "OK $APP OCOR procs="$FNOCOR >> $LOGFILE
fi  
}
checkfilenetlogs()
{
#Check Filenet Elogs
APP=FileNet
SEVERITY="Serious PROBLEM"
ERRORCODE=999
echo $ELOGDIR
lastlog=`ls -t $ELOGDIR | head -1l`
echo $lastlog
read f
if [ -f $ELOGDIR$lastlog ]
#if [ -f $ELOGDIR"elog"$YEAR$MONTH$DAY ]
then
    echo "Check FileNet log files" >> $LOGFILE
#if grep "Fatal error in bes_commit" $ELOGDIR"elog"$YEAR$MONTH$DAY >>
$ELOGTEMPFILE
if grep "Fatal error in bes_commit" $ELOGDIR$lastlog >> $ELOGTEMPFILE
then
    echo "$APP $SEVERITY $ERRORCODE Bes commit not running "$host >>
$LOGFILE
else
    echo "OK $APP bes_commit running "$host >> $LOGFILE
fi
#if grep "'INXbg' is not running" $ELOGDIR"elog"$YEAR$MONTH$DAY >>
$ELOGTEMPFILE
if grep "'INXbg' is not running" $ELOGDIR$lastlog >> $ELOGTEMPFILE
then
    echo "$APP $SEVERITY $ERRORCODE INXbg not running "$host >> $LOGFILE
else
    echo "OK $APP INXbg running "$host >> $LOGFILE
fi  
#if grep "Drive" $ELOGDIR"elog"$YEAR$MONTH$DAY >> $ELOGTEMPFILE
if grep "Drive" $ELOGDIR$lastlog >> $ELOGTEMPFILE
then
    echo "$APP $SEVERITY $ERRORCODE Optical Drive problem "$host >> $LOGFILE
else
    LIBRARYCHECK=`expr $LIBRARYCHECK + 1`
fi
#if grep "disabled" $ELOGDIR"elog"$YEAR$MONTH$DAY >> $ELOGTEMPFILE
if grep "disabled" $ELOGDIR$lastlog >> $ELOGTEMPFILE
then
    echo "$APP $SEVERITY $ERRORCODE Optical Drive disabled "$host >>
$LOGFILE
else
    LIBRARYCHECK=`expr $LIBRARYCHECK + 1`
fi  
#if grep "drivedown" $ELOGDIR"elog"$YEAR$MONTH$DAY >> $ELOGTEMPFILE
if grep "drivedown" $ELOGDIR$lastlog >> $ELOGTEMPFILE
then
    echo "$APP $SEVERITY $ERRORCODE Optical Drive down "$host >> $LOGFILE
else
    LIBRARYCHECK=`expr $LIBRARYCHECK + 1`
fi  
if [ $LIBRARYCHECK = 3 ]
then
    echo "OK $APP Optical library "$host >> $LOGFILE
else
    echo "$APP PROBLEM with Optical library "$host >> $LOGFILE
fi 
#else
#    lastelog=`ls -t $ELOGDIR | head -1l`
    echo "FileNet log files - last elog started on "$lastlog >> $LOGFILE
fi
}
checkbatchjobsrunning()
{
#Check Match_Fetch, Rescan, Print close, are running
APP=BATCH
SEVERITY="Minor PROBLEM"
ERRORCODE="Batch job not running"
echo "Check Batch jobs running "$host >> $LOGFILE
MF=`ps -efa|grep -n Match_Fetch | grep -v grep | awk '{print $5}'|wc -l`
if [ $MF -lt 1 ]
then
    echo "$APP $SEVERITY $ERRORCODE Match_fetch not running. Solution:
Remove stuck case from Staffware queue then restart TWS job "$host >>
$LOGFILE
else
    echo "OK Match_Fetch running "$host >> $LOGFILE
fi
#Check Rescan is running if its after 7am
RS=`ps -efa|grep -n Rescan | grep -v grep | awk '{print $5}'|wc -l`
if [ $RS -lt 1 ]
then
    echo "BING"
#    echo "$APP $SEVERITY $ERRORCODE Rescan not running "$host >> $LOGFILE
else
    echo "OK Rescan running "$host >> $LOGFILE
fi
##Check Print_Close running
PC=`ps -efa|grep -n Print_Close | grep -v grep | awk '{print $5}'|wc -l`
if [ $PC -lt 1 ]
then
    echo "$APP $SEVERITY $ERRORCODE Print close not running "$host >>
$LOGFILE
else
    echo "OK Print close running "$host >> $LOGFILE
fi
}
checkdb()
{
###Check Databases up
APP=DATABASE
SEVERITY="Serious PROBLEM"
ERRORCODE="No response from tns ping"
echo "Check Databases" >> $LOGFILE
for DB in CUCOP1 SWP1 FNTP1
do
    if tnsping $DB | grep OK
    then
      echo "OK "$DB >> $LOGFILE
    else
      echo "$APP $SEVERITY $ERRORCODE $DB tnsping fail, DB not responding!"
$LOGFILE
  fi
done
}
checklighthouseping()
{
APP="Lighthouse3 NT server"
SEVERITY="Serious PROBLEM"
ERRORCODE="No response from ping"
#Check Lighthouse3 responds to ping
echo "Check lighthouse3 NT server response" >> $LOGFILE
if ping lighthouse3 -n 1 | grep "0% packet loss"
# if ping ightouse3 -n 1 | grep "0% packet loss"
  then
      echo "OK $APP ping response" >> $LOGFILE
  else
      echo "$APP $SEVERITY $ERRORCODE ping failed" >> $LOGFILE
      # and if this fails theres no point in attempting an ftp
  fi
}
checkixnping()
{
APP="IXNGATE01 server"
SEVERITY="PROBLEM"
ERRORCODE="No response from ping"
#Check IXNGATE01 responds to ping
echo "Check IXNGATE01 server response" >> $LOGFILE
  if ping ixngate01 -n 1 | grep "0% packet loss"
  then
      echo "OK $APP ping response" >> $LOGFILE
  else
      echo "$APP $SEVERITY $ERRORCODE ping failed" >> $LOGFILE
  fi
}
checkbgta02ping()
{
APP="BGTA02 server"
SEVERITY="PROBLEM"
ERRORCODE="No response from ping"
#Check BGTA02 responds to ping
echo "Check BGTA02 server response" >> $LOGFILE
  if ping bgta02 -n 1 | grep "0% packet loss"
  then
      echo "OK $APP ping response" >> $LOGFILE
  else
      echo "$APP $SEVERITY $ERRORCODE ping failed" >> $LOGFILE
  fi
}
checkstaffwareshutdown()
{
# Check Staffware was shutdown correctly overnight
if [ $HOUR -ge "03" ] -a [ $HOUR -le "23" ]
then
APP=Staffware
SEVERITY="PROBLEM"
ERRORCODE="Staffware was not shutdown"
shutdownday=`cat /usr1/home/pro/checkstaffwaredown.log | awk '{print $1}'`
#shutdownday=`ls -la /usr1/home/pro/checkstaffwaredown.log | awk '{print
$7}'`
#if [ $shutdownday -lt "10" ]
#then
#    shutdownday="0"$shutdownday
#fi
if [ $shutdownday -eq $DAY ]
then
    echo "OK Staffware shutdown correctly " >> $LOGFILE
else
    echo "PROBLEM Staffware was NOT shutdown correctly " >> $LOGFILE
fi
else
    echo "Between 23:00 and 03:00 not checking Staffware shutdown" >>
$LOGFILE
fi
}
checkbatchjobsstarted()
{
#Check Lighthouse scheduled jobs
echo "Check Match fetch, rescan, print close, batch jobs started " >>
$LOGFILE
APP=BATCH
SEVERITY="Minor PROBLEM"
ERRORCODE="Batch job not started"
#LMyyyymmdd RS 
#081120 STARTED MF
#check match fetch started
if grep "STARTED" $LIGHTHOUSELOGDIR"MF"$YEAR$MONTH$DAY.log
then
   echo "OK Match Fetch started" >> $LOGFILE
else
   echo "$APP $SEVERITY $ERRORCODE Match Fetch not started" >> $LOGFILE
fi
#check Rescan started
if grep "STARTED" $LIGHTHOUSELOGDIR"RS"$YEAR$MONTH$DAY.log
then
   echo "OK Rescan started" >> $LOGFILE
else
   echo "$APP $SEVERITY $ERRORCODE Rescan not started" >> $LOGFILE
fi
#check Print Close started
if grep "STARTED" $LIGHTHOUSELOGDIR"PC"$YEAR$MONTH$DAY.log
then
   echo "OK Print close started" >> $LOGFILE
else
   echo "$APP $SEVERITY $ERRORCODE Print close not started" >> $LOGFILE
fi
}
#get Lighthouse3 status file via ftp
#echo "Checking Lighthouse3 NT server status" >> $LOGFILE
# append lighthouse status file to log file
#cat $LIGHTHOUSESTATFILE >> $LOGFILE
#
#! /usr/bin/ksh
############# you can use   ftpyou 2>&1 > /dev/null
############# to suppress output
#HOST=lighthouse3
#USER=fnsw
#PASSWD=fnsw
#exec 4>&1
#ftp -nv >&4 2>&4 |&
#print -p open $HOST
#print -p user $USER $PASSWD
#print -p cd status
#print -p asc
#print -p get $LIGHTHOUSESTATFILE
#print -p bye
#wait
#exit 0
#
checkmemory()
{
ipcs >> $MASTERLOGFILE
vmstat -n >> $MASTERLOGFILE
}
checkdiskspace()
{
echo "Check Disk space" >> $LOGFILE
APP="Disk space"
SEVERITY=PROBLEM
ERRORCODE="Disk space free limit exceeded"
DISKLIMIT=99
flag=0
df -k | grep "%" | awk '{print $1}' > diskspace
for percentage in `cat diskspace`
do
if [ $percentage -gt $DISKLIMIT ]
then
    echo "$APP $SEVERITY $ERRORCODE Disk used at or greater than
$DISKLIMIT%" >> $LOGFILE
    flag=1
fi
done
if [ $flag -lt 1 ]
then
    echo "OK $percentage% Disk space under $DISKLIMIT% limit" >> $LOGFILE
fi 
}
morningstatusmail()
{
# Check whether to send email to Frontline for 7am morning status
# Is it between 7 and and 7:20am?
if [ $HOUR -eq "07" -a $MINUTE -le "10" ]
then
   # if there is already a file then mail has been sent
   if [ -f $MAILSENT ]
   then
       echo "`date` Correspondence system checks, E-mail already sent to
$MORNINGMAILTO" >> $MASTERLOGFILE
   else
       # Else there is no file, so send mail
       echo "`date` Correspondence system checks, sending E-mail" >>
$MASTERLOGFILE
       echo $message > $MESSAGEFILE
       mailx -s "LIGHTHOUSE system status `date`" $MORNINGMAILTO <
$MESSAGEFILE
       > $MAILSENT
   fi
else
   # it's outside of time range, so remove check file to allow it to be
sent next time
   # hour is correct
   if [ -f $MAILSENT ]
   then
       rm $MAILSENT
   fi
fi
}
#
alertmail()
{
#strip out any PROBLEMS and send to PROBFILE for mailing
if grep "PROBLEM" $LOGFILE > $PROBFILE
then
    #1st Copy log file to MASTER log file
    cat $LOGFILE >> $MASTERLOGFILE
    echo "Please call Lighthouse support on 07769 548782" >> $PROBFILE
    mailx -s "**LIGHTHOUSE PROBLEM** `date`" $MAILTO < $PROBFILE
else
    echo "`date` All system checks OK" >> $MASTERLOGFILE
fi
}
# BLACKOUT PERIOD - 03:00 - 03:30 is when apps get shutdown restarted
# so we dont want alerts during this time
# Check time, if its 03:00 to 03:30 filenet and staffware get restarted
if [ $HOUR -eq "03" -a $MINUTE -le "35" ]
then
  echo "`date` Not checking system during shutdown restart times" >>
$LOGFILE
  echo "`date` Not checking system during shutdown restart times" >>
$MASTERLOGFILE
else
checkfilenet()
{
checkfilenetprocs
}
checkday()
{
dayname=`date +%a`
if [ $dayname = "Sat" -o $dayname = "Sun" ]
then
    echo "`date` Not checking batch jobs on Sat/Sun" >> $LOGFILE
    echo "`date` Not checking batch jobs on Sat/Sun" >> $MASTERLOGFILE
    DOCHECK=0
else
    DOCHECK=1
fi
} 
checktime()
{
#if [ $HOUR -gt $BS -o $HOUR -lt $BE ]
if [ $HOUR -gt $BS -o $HOUR -lt "09" ]
then
    echo "`date` Not checking batch jobs between $BS":00" and $BE":00"" >>
$LOGFILE
    echo "`date` Not checking batch jobs between $BS":00" and $BE":00"" >>
$MASTERLOGFILE
    DOCHECK=0
else
    DOCHECK=1
fi
}
checkcapacity()
{
echo "Check capacity monitor" >> $LOGFILE
APP="Capacity monitor"
SEVERITY="Minor PROBLEM"
ERRORCODE="Capacity monitor not running! To restart logon to hsd111/151 as
fnsw, then /fnsw/local/fnsw_home/startcapacity"
if procname=`ps -efa|grep "capacity"| grep -v grep`
then
    echo "OK capacity monitor running" >> $LOGFILE
else
    echo "$APP $SEVERITY $ERRORCODE" >> $LOGFILE
fi
}
checklogfilesize
checkdb
checkfilenet
checkfilenetcache
#checkfilenetlogs
checkstaffwareprocs
#checkstaffwareshutdown
checkday
if [ $DOCHECK -gt 0 ] 
then
    checktime
    if [ $DOCHECK -gt 0 ] 
    then
        #checkbatchjobsstarted
        checkbatchjobsrunning
    fi
fi
checklighthouseping
checkixnping
checkbgta02ping
#checkmemory
checkdiskspace
checkcapacity
alertmail
if grep "Serious PROBLEM" $LOGFILE
# If there are any 'Serious' problems we should NOT be sending an Ok
morning status email
# 
then
   echo "Problems were found in logfile, so Morning Status email was not
sent" >> $LOGFILE
   echo "Problems were found in logfile, so Morning Status email was not
sent" >> $MASTERLOGFILE
# Otherwise DO send a Morning status email
else
   morningstatusmail
fi
fi
 
 
/opt/product/scripts/users/scripts/syscheckwrapper run as swadmin
 
# Wrapper for running batch monitor script
# To run use       nohup ./syscheckwrapper &
while true
do
  ./syscheck
  #wait in seconds 
  sleep 360
done
 
 
/opt/product/scripts/users/scripts/checksyscheck run as swadmin
 
/opt/product/scripts/users/scripts/checksyscheck run as swadmin
# Julian Ward 03/02/2009 - checksyscheck
# Used to check and restart 'syscheck' system monitor
# Runs every INTERVAL mins and checks whether the syslog timestamp is
# within MINUTELIMIT mins of system time. If not it restarts the
syscheckwrapper
# To run use    nohup ./checksyscheck &
#
# Get existing interval from syscheckwrapper script
SYSLOOP=`grep "sleep" syscheckwrapper | awk '{print $2}'`
SYSMINS=`expr $SYSLOOP "/" 60`
# Set minutelimit to be +2 mins greater than the time interval syscheck runs
MINUTELIMIT=`expr $SYSMINS "+" 2`
# Set run interval for this script to be greater than the interval syscheck
runs
INTERVAL=`expr $MINUTELIMIT "+" 2`
INTERVALSECS=`expr $INTERVAL "*" 60`
HOURLIMIT=1
LOGFILE="/opt/product/scripts/users/logs/syscheck.log"
MASTERLOGFILE="/opt/product/scripts/users/logs/syscheckmaster.log"
restart()
{
#kill existing process
procid=`ps -efa|grep syscheckwrapper| grep -v grep | awk '{print $2}'`
kill $procid
echo `date`" Restarting syscheck monitor" >> $MASTERLOGFILE
nohup /opt/product/scripts/users/scripts/syscheckwrapper &
}
getfiletime()
{
SYSHOUR=`date +%H`
SYSMINUTE=`date +%M`
FILEHOUR=`ls -la $LOGFILE | awk '{print $8}'| cut -c1-2`
FILEMINUTE=`ls -la $LOGFILE | awk '{print $8}'| cut -c4-6`
# is there more than a $MINUTELIMIT difference in time
HOURDIFF=`expr $SYSHOUR - $FILEHOUR`
#MD=`expr $SYSMINUTE - $FILEMINUTE`
#echo $MD
#read f
MINUTEDIFF=`expr $SYSMINUTE - $FILEMINUTE`
#echo "hour dif minute diff"
#echo $HOURDIFF $MINUTEDIFF $MINUTELIMIT
#read x
if [ "$MINUTEDIFF" -gt "$MINUTELIMIT" ] -o [ "$MINUTEDIFF" -lt "0" ]
then
    restart
fi
#echo "System time " $SYSHOUR":"$SYSMINUTE
#echo "File time   " $FILEHOUR":"$FILEMINUTE
}
while true
do
  getfiletime
  #interval to run this check
  # 
  sleep $INTERVALSECS
done