#!/usr/bin/ksh ##################################################################################################### # Developed by Shailesh Patel V1.0 - 3/10/09 # # # This script does the SRDF link check and the output is to be parsed by a differnet perl script # and upload it to the web. ##################################################################################################### ####################################################################################################################### # Prerequiste: # A. The script requires $LINPUT file to get the information which is output of the get_local_arrays script. # Thus the get_local_arrays script has to be run once every day before this current script runs. # B. The current script has to be run on every unix control workstation in a gap of 3-4 minutes in same hour. # # The flow of the script is as follows # 1. Check to see if the script is run as root or normal id nad chnage the pbrun variable accordingly # 2. Check to see the Exclude file is present $Exclude_list to exclude array from checking the SRDF link status. # 3. Check to see the Arrays with SRDF have RF which are online but no link connections exists. # 4. For Total RF Link falure for an array alert Tivoli ( adapter not yet set). # 5. Email the status report or $OUT find the Error file to the DSM group # 6. the $OUT file will be parsed by a different perl script to project the status on the web. # ####################################################################################################################### set -x DIR=/gsmsan04/cob_scripts/srdf_link_config LOG=/gsmsan04/cob_logs/srdf_link_logs Exclude_list=$DIR/exclude_array_check Rfile=$DIR/site_replacement INPUT=$DIR/array_relationship OLINPUT=$DIR/local_arrays_`date +"%m%d%y"` LINPUT=$DIR/local_arrays TMP0=/tmp/tmp0.$$ TMP1=/tmp/tmp1.$$ TMP2=/tmp/tmp2.$$ TMP3=/tmp/tmp3.$$ TMP4=/tmp/tmp4.$$ #TMP4=/tmp/symm3668_test #TMP4=/tmp/symm1415_test TMP5=/tmp/tmp5.$$ TMP6=/tmp/tmp6.$$ TMP7=/tmp/tmp7.$$ TMP8=/tmp/tmp8.$$ email_list=shailesh.r.patel@citigroup.com #email_list=gt.us.distributed.storage.mgmt@imcnam.ssmb.com ################################################## Usage() { #set -x #tput clear [[ -n "$1" ]] && print "\n\n$(basename $0): $@ \n " print "\n\n $(basename $0) -s \n" print " arguments: " print " -s : Indicates to start the SRDF link check for the Symmetrix \n " print " \n\n sample usage is $(basename $0) -s start \n\n" print "\t The Output file is ${LOG}/status_srdf_link_ \n" print "\t This script does the SRDF link check and generates a file to be used to report on the web \n" exit 1 } os_type() { set -x os=`/usr/bin/uname` osver=`/usr/bin/uname -a|awk '{print $3}'` case $os in SunOS) export PATH=/usr/xpg4/bin:$PATH if [[ "$osver" = "5.8" ]];then wgrep=`echo "grep -Ew "` else wgrep=`echo "grep -w "` fi export wgrep ;; AIX) wgrep=`echo "egrep -w "` export wgrep ;; esac } Arg() { set -x [[ $# = 0 ]] && Usage while getopts :s: OPT; do case $OPT in s) check=$OPTARG ;; \?) Usage $@ ;; esac done shift OPTIND-1 if [[ "$check" != "start" ]];then Usage "Error: Incorrect argument used on `hostname` : Please check the argument" fi tag=`id |egrep uid | sed 's/=/ /g'|awk '{print $2}'|awk -F'(' '{print $1}'` if [[ "$tag" -eq 0 ]];then vpbrun="" else vpbrun="pbrun" fi host=`hostname` OUT=$LOG/status_srdf_link_`date +"%m%d%y_%H"00`.out } Date() { set -x print "\n*******************************************************************************************" print " `hostname`: $1 : `date +"%D,%T"` " print "*******************************************************************************************\n" } ############################################################# # Gather the Information about R1 arrays only ############################################################# # 1. Check to see the Arrays has RDF reationship # 2. Check to see if it is R1 array the proceed # 3. Check to see if it all the RF links are UP Main(){ set -x cp -p $OLINPUT ${LINPUT} if [[ -s "${Exclude_list}" ]];then exclude_array=`cat ${Exclude_list}|egrep -v "^#" | $wgrep "^${host}" |awk -F':' '{print $2}'` if [[ "${exclude_array}" != "" ]];then exclude=`echo $exclude_array|sed 's/ /|/g'` $vpbrun symcfg list |egrep Local |awk '{print $1}'|egrep -v "$exclude" |sed -e "s/^/$host:/g" > $TMP1 dos2unix $TMP1 $TMP1 >/dev/null 2>&1 else $vpbrun symcfg list |egrep Local |awk '{print $1}' |sed -e "s/^/$host:/g" > $TMP1 dos2unix $TMP1 $TMP1 >/dev/null 2>&1 fi else $vpbrun symcfg list |egrep Local |awk '{print $1}' |sed -e "s/^/$host:/g" > $TMP1 dos2unix $TMP1 $TMP1 >/dev/null 2>&1 fi for ssymm in `cat $TMP1 |awk -F':' '{print $2}'` do $vpbrun symcfg -sid $ssymm -ra all list >/dev/null 2>&1 if [[ "$?" -eq 0 ]];then $vpbrun symrdf -sid $ssymm list >$TMP2 dos2unix $TMP2 $TMP2 >/dev/null 2>&1 cat $TMP2 |egrep -v "pam_dhkeys"|sed -n '11,11p'|egrep "R1:" >$TMP3 if [[ -s $TMP3 ]];then ################################### # To test manually, comment below line ################################### $vpbrun symcfg -sid $ssymm -ra all list >$TMP4 dos2unix $TMP4 $TMP4 >/dev/null 2>&1 cat $TMP4 |egrep Online|awk '{print $7}' |sort -du >$TMP5 count=`cat $TMP5 |wc -l |awk '{print $1}'` var_check=`cat $TMP5|egrep "^-"` for tsymm in `cat $TMP5` do if [[ "$tsymm" = "-" ]];then DRF=`cat $TMP4|egrep Online |egrep " - -"|awk '{print $1}'` Down_RF=`echo $DRF|sed 's/ /, /g'` var=`print "$host:$ssymm" |sed -f ${Rfile}` ctsymm=`$wgrep "^$var" $INPUT |awk -F':' '{print $6}'` if [[ "$ctsymm" = "" ]];then print "Error: For $var there is No Target Symm in $INPUT file" >>$OUT else for ttsymm in $ctsymm ;do ################################################################ # To test total RF failure, comment below line and put dummy symm id ################################################################ $vpbrun symrdf -sid $ttsymm -rdf ping >/dev/null 2>&1 #$vpbrun symrdf -sid 0858 -rdf ping >/dev/null 2>&1 if [[ "$?" -eq 0 ]];then tvar=`$wgrep "^$var" $INPUT |$wgrep "$ttsymm"` print "`date +"%m/%d/%y_%H:%M_"`:$tvar:Partial_RF_Link_Down:for $Down_RF" >>$OUT else tvar=`$wgrep "^$var" $INPUT |$wgrep "$ttsymm"` ttvar=`$wgrep "^$var" $INPUT |$wgrep "$ttsymm"|awk -F':' '{print $4":"$5":"$6}'` print "`date +"%m/%d/%y_%H:%M_"`:$tvar:ALL_RF_Link_Down:for $Down_RF" >>$OUT /bin/logger -p local0.err -t SRDF_ALL_LINK_DOWN "Please contact SAN Support. ALL RF LINK DOWN between $var Symmetrix and $ttvar Symmetrix ." print " \n\n SRDF_ALL_LINK_DOWN : Please contact SAN Support. ALL RF LINK DOWN between $var Symmetrix and $ttvar Symmetrix. \n\n" >$TMP8 ############################ #Actual Alert in Tivoli ############################ ##### /bin/logger -p local0.err -t SRDFA_INACTIVE_ALERT "Please contact SAN Support. SRDF/A session for RDF group ${rdfgroup} on Symmetrix $ssid is inactive" # /bin/logger -p local0.err -t SRDFA_INACTIVE_ALERT "Please contact SAN Support. ALL RF LINK DOWN between $ssymm Symmetrix and $ttsymm Symmetrix ." fi OUT1=$LOG/${ssymm}_detail_list_`date +"%m%d%y_%H"`.err Date "$ssymm" > $OUT1 cat $TMP8 >> $OUT1 $vpbrun symcfg -sid $ssymm -ra all list -port >>$OUT1 $vpbrun symcfg -sid $ssymm -rdfg all list >>$OUT1 $vpbrun symcfg -sid $ssymm -rdfg all list -rdfa >>$OUT1 print "$OUT1" >> $TMP6 done fi elif [[ "$count" -eq 1 ]] || [[ "$count" -eq 2 ]] && [[ "$var_check" = "" ]];then var=`print "$host:$ssymm" |sed -f ${Rfile}` present=`$wgrep "$var" $INPUT |$wgrep "$tsymm"` if [[ "$present" = "" ]];then avar=`$wgrep "$tsymm" $LINPUT|head -1|sed -f ${Rfile}` print "$var:$avar" >>$INPUT fi $vpbrun symrdf -sid $tsymm -rdf ping >/dev/null 2>&1 if [[ "$?" -eq 0 ]];then tvar=`$wgrep "^$var" $INPUT |$wgrep "$tsymm"` print "`date +"%m/%d/%y_%H:%M_"`:$tvar:ALL_RF_Link_UP:" >>$OUT #else # This situation might not arise whereby you have the Target Symm id # in symcfg -ra all list but you cannot ping the Targt symm # tvar=`$wgrep "^$var" $INPUT |$wgrep "$tsymm"` # print "$tvar:ALL_RF_Link_Down:for $Down_RF" >>$OUT ######## Alert Tivoli######## fi fi done fi fi done if [[ -s $TMP6 ]];then for err in `cat $TMP6`;do cat $err >>$TMP7 done fi if [[ -s $TMP7 ]];then cat $TMP7 |mailx -s "Error: Status of SRDF Arrays with Partial or All RF down from $host @ `date +"%m/%d/%y_%H":00` " $email_list fi if [[ -s $OUT ]];then cat $OUT |mailx -s "Status of SRDF Link Check from $host @ `date +"%m%d%y_%H":00` " $email_list fi } Clean(){ rm /tmp/tmp*.$$ } os_type Arg $@ Main Clean