#!/usr/bin/perl
# global_functions.pm
###################################################################
# Perl Global Functions
#
###################################################################
#
# Version History
# ~~~~~~~~~~~~~~~
#
# Date        Version  Author           Description
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# 06.08.2009   0.1     Ian Preece        Initial version
#
###################################################################
use warnings;
use strict;


# Internal HDS subroutine declaration
 sub raidqry_details;                                   # Raidqry -g
 sub pair_display;                                      # pairdisplay -g -fxc -CLI
 sub check_SVOL_attribute;                              # Sift out SVOL info from pairdisplay
 sub check_pair_status;                                 # pairevtwait -g -nowait
 sub wait_pair_status;                                  # pairevtwait -g
 sub pair_volchk;                                       # Use pairvolchk to check local status only
 sub match_pair_details;                                # Sift out info from pairdisplay
 sub wait_pair_status_old;                              # Not currently used
 sub journal_usage;                                     # Displays Journal Usage
 sub get_date;                                          # Gets current date
 sub get_time;                                          # Gets current time
 sub log_message;                                       # Writes a message to screen and the log file
 sub file_exists;                                       # returns 0 if the passed file exists and 1 if it does not
 sub check_file;                                        # Searches for a string in a file
 sub clean_up;                                          # Clean up before exitting
 sub lock_devgroup;                                     # Lock a device group 
 sub unlock_devgroup;                                   # Unlock a device group
 sub list_devgroups;                                    # return a list of all the device groups in the horcm file
 sub raise_alarm;                                       # generate an alarm using logger
 sub RPO_timeout;                                       # Checks if the passed device group exceeds the passed RPO timeout value
 sub get_cci_cmd;                                       # Generates the POWERBROKER Command if necessary
 sub pool_usage;					# Generates pool usage information
 sub set_horcminstance;					# set horcm instance according to the site config file
 sub set_env;						# Set the HORCM Environment variables
 sub check_HORCM;					# Check to see whether the HORCM Instance is valid by checking if the HORCM file exists
 sub check_location;					# Confirm location (i.e. P-VOLs at the local or remote site)
 sub exit_error;					# Control the error handling and exitting
 sub execute_com;  					# Execute the command
 sub is_si_local;					# Determine if the ShadowImage group is local
 
##########################################################################
sub get_cci_cmd
{
 my $cci_cmd = $_[0];
 my $ret;

 if (exists($ENV{'POWERBROKER'})){
   my $pbrun = $ENV{'POWERBROKER'};
   unless ( -x $pbrun) {
    print "POWERBROKER env variable incorrectly set\n";
    exit(1);
   }
   if (exists($ENV{'HORCC_MRCF'})) {
       $ret = $pbrun." ".$cci_cmd." -ISI$ENV{'HORCMINST'} ";
   }
   else {
       $ret = $pbrun." ".$cci_cmd." -I$ENV{'HORCMINST'} ";
   }

 }
 else {
  $ret = $cci_cmd;
 }
 return $ret;
}

##########################################################################
sub get_date 
{
  my $Sep = shift||"\/";                                                                        #Separator (e.g. '/', '_', ':')
  my @months = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
  (my $sec,my $min,my $hour,my $mday,my $mon,my $year,my $wday,my $yday,my $isdst)=localtime(time);
  $year=$year+1900;
  my $getdate = "$mday$Sep$months[$mon]$Sep$year";

  my $retkey = shift || "";
  if ($retkey ne "n") {
    return $getdate;
  } else {
      return "$getdate\n";
  }
}

##########################################################################
sub get_time 
{
  my $Sep = shift||":";                                                                         #Separator (e.g. '/', '_', ':')
  (my $sec,my $min,my $hour,my $mday,my $mon,my $year,my $wday,my $yday,my $isdst)=localtime(time);
  my $gettime = "$hour$Sep$min$Sep$sec";

  my $retkey = shift || "";
  if ($retkey ne "n") {
    return $gettime;
  } else {
      return "$gettime\n";
  }
}

##########################################################################
sub log_message 
{
# Used to log the from the HDS scripts
# monitor script format = <DATE+TIME>,<INFO|WARN|ERROR>,<REPL_GROUP>,<ERROR>
# control script format = <DATE+TIME>,<USERID>,<OPERATION>,<REPLICATION_GROUP>

  my $gd = get_date();
  my $gt = get_time();
  my $message = shift || exit_error("Log message error: blank message", "ERR_LOG_NO_MESS");
  my $logfile = shift || exit_error("Log message error: unspecified file", "ERR_LOG_NO_FILE");
  my $loglevel = shift || 2;

  my $loglevel_hash={};
  $loglevel_hash->{ '0' } = 'ERROR';
  $loglevel_hash->{ '1' } = 'WARN';
  $loglevel_hash->{ '2' } = 'INFO';


  my $time =  "$gd $gt";
  print "$message\n";
  open (LOGF,">>$logfile")or die $!;
  if ( $main::SCRIPT_TYPE eq "control" ) {
        print LOGF "$time,$main::USERNAME,$main::FULLNAME,$main::DEVGROUP,$message\n";
  }
  elsif ( $main::SCRIPT_TYPE eq "monitor" ) {
        print LOGF "$time,$loglevel_hash->{$loglevel},$main::DEVGROUP,$message\n";
  }
  close LOGF;
}

##########################################################################
sub raise_alarm 
{
 my $message = shift;
 my $support_team="SAN Support Team";
 my $RC=system("\/bin\/logger -p local0.err -t HUR_ALERT \"Please contact $support_team. $message\"");
}

##########################################################################
sub file_exists 
{
 my $file = shift ||exit_error ("File exists error: No file name specified", "ERR_FILE_NO_NAME");

 if ( -e $file && -f $file ){
   return 0;
 } else {
     exit_error ("File exists error: '$file' does not exist", "ERR_FILE_NO_EXIST");
     return 1;
   }
}

##########################################################################
sub check_file 
{
 my $filename = shift || exit_error("File search error: no file specified", "ERR_FILE_NO_OPEN");
 my $search_str = shift || exit_error("File search error: no searh string specified", "ERR_FILE_NO_OPEN");
 my $found = "not found";

 open FH, $filename|| exit_error("File search error: cannot open file $filename", "ERR_FILE_NO_OPEN");

 while (<FH>) {
  if (/$search_str/) {
    close FH;
    return 0;
  }
 }
close FH;
return 1;
}

##########################################################################
sub is_si_local 
{

 my $GROUP = shift || exit_error("is_si_local error: No Device Group passed to routine","ERR_RAIDQRY_NO_GRPDEF");   # Device Group
 my $R_VAL = -1;                                        # Local return code variable

# Generate full command path
 my $CMD = $main::HORCM_CLIDIR . "raidqry";
 my $CCICMD = get_cci_cmd($CMD);
 my $FULLCOMM = $CCICMD . " -r " . $GROUP;

# Execute the command
 my @ALL_LINE = qx($FULLCOMM 2>/dev/null);
 $R_VAL = $?/256;
 if ( $R_VAL == 251 ) {
   exit_error("Raidqry error: raidqry -r $GROUP failed. Horcm instance could possibly be down. Please check that horcm instance $main::HORCMINST is up.",$R_VAL, 1, 1);
 }
 elsif ( $R_VAL == 239 ) {
   return 1;              # ShadowImage group is not local
 }
 elsif ( $R_VAL == 0 ) {
   return 0;              # ShadowImage group is local
 }
 else {
   exit_error("Unable to determine if shadowimage group $GROUP is local to $main::HOSTNAME");
 }
    
}

##########################################################################
sub raidqry_details 
{

 my $GROUP = shift || exit_error("Raidqry error: No Device Group passed to routine","ERR_RAIDQRY_NO_GRPDEF");   # Device Group
 my $R_VAL = -1;                                        # Local return code variable

# Generate full command path
 my $CMD = $main::HORCM_CLIDIR . "raidqry";
 my $CCICMD = get_cci_cmd($CMD);
 my $FULLCOMM = $CCICMD . " -r " . $GROUP;

# Execute the command
 my @ALL_LINE = qx($FULLCOMM);
 $R_VAL = $?/256;
 if ( $R_VAL != 0 ) {
   exit_error("Raidqry error: raidqry -r $GROUP failed. Horcm instance could possibly be down. Please check to see if both instances are up.",$R_VAL, 1, 1);
 }
 else {
   log_message( "Device Group $GROUP validated by raidqry -r" , $main::LOGFILE);
   return 0;
 } 

}

##########################################################################
sub pair_display 
{

 my $GROUP = shift||exit_error("Pair display error: Missing parameter","ERR_PAIRDIS_NO_DET");            # Device Group;
 my $LOCAL = shift||0;                                   # 1 = 'pairdisplay -l' (local only)
 my $FULLCOMM = "";
 my $R_VAL = -1;                                 # Local return code variable

 if ( $LOCAL == 1 ) {
   my $CMD= $main::HORCM_CLIDIR . "pairdisplay";
   my $CCICMD=get_cci_cmd($CMD);
   $FULLCOMM = $CCICMD ." -g " . $GROUP . " -fxc -CLI -l";
   @main::PAIRDISPLAY = qx($FULLCOMM);
   $R_VAL = $?/256;
   if ( $R_VAL > 3 ) {
     exit_error("Check pair display error: pairdisplay -l command generated an error: $FULLCOMM. The local HORCM instance may be down.", $R_VAL, 1, 1);
   }
 }
 elsif ( $LOCAL == 2 ) {
   my $CMD= $main::HORCM_CLIDIR . "pairdisplay";
   my $CCICMD=get_cci_cmd($CMD);
   $FULLCOMM = $CCICMD ." -g " . $GROUP . " -fxc -CLI -FMRCF 1";
   @main::PAIRDISPLAY = qx($FULLCOMM);
   $R_VAL = $?/256;
   if ( $R_VAL > 3 ) {
     exit_error("Check pair display error: pairdisplay command generated an error: $FULLCOMM. The local HORCM instance may be down.", $R_VAL,
 1, 1);
   }
 }

 else {
  log_message("Checking pair current status.", $main::LOGFILE);

  my $CMD= $main::HORCM_CLIDIR . "pairdisplay";
  my $CCICMD=get_cci_cmd($CMD);
  $FULLCOMM = $CCICMD . " -g " . $GROUP . " -fxc -CLI";
  @main::PAIRDISPLAY = qx($FULLCOMM);
  $R_VAL = $?/256;
  if ( $R_VAL > 3 ) {
    $main::LMODE = 1;
    pair_display ($GROUP, $main::LMODE);
    log_message("Error checking pair status '$FULLCOMM'.Now issuing '$FULLCOMM -l'", $main::LOGFILE);
    log_message("Unable to communicate with a HORCM instance. The remote HORCM instance may be down.", $main::LOGFILE);
  }
 }

 $FULLCOMM = "\n@main::PAIRDISPLAY";
 chomp $FULLCOMM;
 log_message($FULLCOMM, $main::LOGFILE);

 return $R_VAL;

}

##########################################################################
# Check S-VOL Attributes
##########################################################################
# Confirm S-VOL Read/Write Attributes (i.e. Read/Only or Write Enabled, or has been modifed)
sub check_SVOL_attribute
{

 my $MSG = "";
 my $R_VAL = -1;                                # Local return code variable
 my $MATCH = 0;                                 # Local return code variable


# Check if SVOLS in SSWS state
 $MATCH = match_pair_details($main::DEVGROUP, "S-VOL", "SSWS", 1);

 if( $MATCH == 1 ) {
   $MSG = "Secondary (S-VOL) volumes are in a suspended Write Enabled State with dominant delta (SSWS)";
   $R_VAL = 4;
   log_message($MSG, $main::LOGFILE);
   return $R_VAL;
 }


# Check if SVOLS in RO or RW state
 $MATCH = match_pair_details($main::DEVGROUP, "P-VOL", " W\n", 1);

 if( $MATCH == 1 ) {
   $MSG = "Secondary (S-VOL) volumes are in a Write Enabled State";
   $R_VAL = 2;
 } elsif ($MATCH == 2) {
     $MSG = "Secondary (S-VOL) volumes are in a Read Only State";
     $R_VAL = 0;
   }

# log_message($MSG, $main::LOGFILE);


# Check if SVOLS have been modified
 $MATCH = match_pair_details($main::DEVGROUP,"S-VOL", " W\n", 1);

 if ($MATCH == 1) {
   $MSG = "Secondary (S-VOL) volumes have been modified since been suspended";
   $R_VAL = $R_VAL + 1;
 } elsif ($MATCH == 2) {
     $MSG = "Secondary (S-VOL) volumes have not been modified since been suspended";
   }

 log_message($MSG, $main::LOGFILE);

 return $R_VAL;
}

##########################################################################
# Check_status
##########################################################################
sub check_pair_status {

my $GROUP = shift||exit_error("Pair event wait error: Missing parameter","ERR_PAIRWAIT_NO_DET");                # Device Group;
my $LOCAL = shift||0;                                   # 1 = 'pairdisplay -l' (local only)
print "check_pair_status : local is set to $LOCAL\n";
print "check_pair_status : group is set to $GROUP\n";
my $R_VAL = -1;                                 # Local return code variable

if ( $LOCAL == 1 ) {
  $R_VAL  = pair_volchk ($GROUP);                                               # Use pairvolchk for local side only
 # print "pairvolchk return is: " . statuslist($R_VAL) . "\n";
} else {
# Generate full command path
    my $CMD = $main::HORCM_CLIDIR . "pairevtwait";
	print "DEBUG : CMD is set to $CMD\n";
    my $CCICMD = get_cci_cmd($CMD);
	print "DEBUG : CCICMD is set to $CMD\n";
    my $REMOTE_OPTION="";
    if ( $LOCAL == 2 ) {
       $REMOTE_OPTION=" -FMRCF 1 ";
    }
    my $FULLCOMM = $CCICMD . " -g " . $GROUP . " -nowait -nomsg $REMOTE_OPTION";
	print "debug - my command is $FULLCOMM\n";
    print qx($FULLCOMM);
    print "DEBUG PRE : R_VAL is $R_VAL\n";
    $R_VAL =$?/256;
    print "DEBUG POST : R_VAL is $R_VAL\n";

    if ( $R_VAL > 5 ) {
      if ( $R_VAL == 242 ) {
        log_message("Error checking pair status '$FULLCOMM'.Now issuing '$FULLCOMM -l'", $main::LOGFILE);
        $main::LMODE = 1;
        check_pair_status($GROUP, $main::LMODE);
      } else {
          log_message("Error checking pair status '$FULLCOMM'.\n\t Pair details displayed in $main::LOGFILE",  $main::LOGFILE);
          log_message("See error log file '$main::ERRFILE'.",  $main::LOGFILE);
          log_message("pairdisplay -l has now been sent", $main::LOGFILE);
          pair_display ($GROUP, 1);
          exit_error("Check pair status error: pairevtwait command generated an error: $FULLCOMM", $R_VAL, 1, 1);
          }
    }
  }
print "DEBUG : return value is $R_VAL\n";

return $R_VAL;

}

##########################################################################
# Wait_status
##########################################################################
sub wait_pair_status {

my $GROUP = shift||exit_error("Pair event wait error: Missing parameter","ERR_PAIRWAIT_NO_DET");                # Device Group
my $STATE = shift||exit_error("Pair event wait error: Missing parameter","ERR_PAIRWAIT_NO_DET");                # State
my $TIMEOUT = shift ;
my $R_VAL = -1;                                 # Local return code variable
my $INTERVAL = 600;                             # Wait interval 600 seconds
my $ENDLOOP = 0;                                # Trigger to end do loop
my $TTOTAL = 0;                                 # Total accumulated wait time (s)
my $h = 0;                                      # Accumulated wait time in integer hours
my $m = 0;                                      # Accumulated wait time in modulo minutes

# Generate full command
my $REMOTE_OPTION="";
if ( $main::LMODE == 2 ) {
  $REMOTE_OPTION=" -FMRCF 1";
}
my $CMD = $main::HORCM_CLIDIR . "pairevtwait";
my $CCICMD = get_cci_cmd($CMD);
my $FULLCOMM = $CCICMD . " -g " . $GROUP . " -s " . $STATE . " -nomsg" . $REMOTE_OPTION;

if ( $main::LMODE == 1 ) {
  $FULLCOMM = $FULLCOMM . " -l";
}



if ( $TIMEOUT > 0 ) {
  do {
    if ( $TIMEOUT > $INTERVAL ) {
      $FULLCOMM = $FULLCOMM . " -t " . $INTERVAL;
      print qx($FULLCOMM);                              # Repeat every interval till $TIMEOUT < $INTERVAL
      $R_VAL =$?/256;
      $TIMEOUT = $TIMEOUT - $INTERVAL;
      $TTOTAL = $TTOTAL + $INTERVAL;
      if ( $R_VAL != 232 && $R_VAL != 233) {
        $ENDLOOP = 1;
      } else {
          $h = sprintf("%.f", $TTOTAL/3600);
          $m = sprintf("%.2f", ($TTOTAL%3600)/60);
          if ( $TIMEOUT > $INTERVAL ) {
            log_message("Pair state '$STATE' not yet achieved in $TTOTAL seconds ($h hours $m minutes).\n\t\t\tWaiting another $INTERVAL seconds", $main::LOGFILE);
          } else {
              log_message("Pair state '$STATE' not yet achieved in $TTOTAL seconds ($h hours $m minutes).\n\t\t\tWaiting a final $TIMEOUT seconds", $main::LOGFILE);
            }
          pair_display ($GROUP, $main::LMODE);
        }
    } else {
        $FULLCOMM = $FULLCOMM . " -t " . $TIMEOUT;
        print qx($FULLCOMM);                            # Last remaining time left
        $R_VAL =$?/256;
        $TTOTAL = $TTOTAL + $TIMEOUT;
        $ENDLOOP = 1;
      }
  } until $ENDLOOP == 1; ;
} else {                                                                                # Infinite loop till a non-time out return (232 or 233) is received
    do {
      $FULLCOMM = $FULLCOMM . " -t " . $INTERVAL;
      print qx($FULLCOMM);
      $R_VAL =$?/256;
      $TTOTAL = $TTOTAL + $INTERVAL;
      if ( $R_VAL != 232 && $R_VAL != 233) {
        $ENDLOOP = 1;
      } else {
          $h = sprintf("%.f", $TTOTAL/3600);
          $m = sprintf("%.2f", ($TTOTAL%3600)/60);
          log_message("Pair state '$STATE' not yet achieved in $TTOTAL seconds ($h hours $m minutes).\n\t\t\tWaiting another $INTERVAL seconds", $main::LOGFILE);
          pair_display ($GROUP, $main::LMODE);
        }
    } until $ENDLOOP == 1 ;
  }

#print "Status = $R_VAL\n";
if ( $R_VAL != 0 ) {
  if ( $R_VAL == 232 || $R_VAL == 233) {
   log_message("Timeout waiting for pair to change to '$STATE' status.", $main::LOGFILE);
   log_message("pairdisplay -l has now been sent", $main::LOGFILE);
   pair_display ($GROUP, 1);
   exit_error("Check pair status error: Timeout waiting for pair to change to '$STATE' status after command: $FULLCOMM", $R_VAL, 1, 1);
  } else {
   log_message("Error waiting for pair status.\n\t Pair details displayed in $main::LOGFILE", $main::LOGFILE);
   log_message("pairdisplay -l has now been sent", $main::LOGFILE);
   pair_display ($GROUP, 1);
   exit_error("Check pair status error: pairevtwait command generated an error: $FULLCOMM", $R_VAL, 1, 1);
  }
}
return $R_VAL;

}

##########################################################################
# Use pairvolchk to check local status only
##########################################################################
sub pair_volchk {                                                       # Used if the remote HORCM is down

my $GROUP = shift||exit_error("Pair event wait error: Missing parameter","ERR_PAIRWAIT_NO_DET");                # Device Group;
my $R_VAL = -1;                                                 # Local return code variable

# Generate full command
 my $CMD = $main::HORCM_CLIDIR . "pairvolchk";
 my $CCICMD = get_cci_cmd($CMD);
 my $FULLCOMM = $CCICMD . " -g " . $GROUP . " -ss -nomsg";


print qx($FULLCOMM);
$R_VAL = $?/256;

$R_VAL = evtlist ($R_VAL);                                      # Convert to pairevtwait return codes

  if ( $R_VAL > 5 ) {
    log_message("Error checking pair status '$FULLCOMM'.\n\t Pair details displayed in $main::LOGFILE", $main::LOGFILE);
    log_message("See error log file '$main::ERRFILE'.", $main::LOGFILE);
    log_message("pairdisplay -l has now been sent", $main::LOGFILE);
    pair_display ($GROUP, 1);
    exit_error("Check pair status error: pairevtwait command generated an error: $FULLCOMM", $R_VAL, 1, 1);
  }

return $R_VAL;

}

##########################################################################
# Check if pairsyncwait return before the passed threshold
##########################################################################
sub RPO_timeout {

my $GROUP = shift;
my $TIME  = shift;

# Generate full command path
my $CMD = $main::HORCM_CLIDIR . "pairsyncwait";
my $CCICMD = get_cci_cmd($CMD);
my $FULLCOMM = $CCICMD . " -g " . $GROUP . " -t $TIME -nomsg";

my $R_VAL=system($FULLCOMM);

return $R_VAL;

}

##########################################################################
sub pool_usage {


# find the number of RAID units specified in the horcm file
# each raid unit appears in the HORCM_CMD section of the horcm file contains 'dev' and does not start with #
 my $horcmfile= $main::HORCM_DIR . "horcm" . $main::HORCMINST . ".conf";
 my $MARKER=0;
 my $NO_RAID_UNINTS=0;
 open (HORCM,"<$horcmfile");
 while ( my $line = <HORCM> ) {
    chomp $line;
    if (( $line =~ /^HORCM_DEV/ ) || ($line =~ /^HORCM_LDEV/)) {
        $MARKER=0;
    }
    if (( $MARKER == 1 ) && ( $line !~ /^#/ ) && ( $line =~ /dev/)) {
        $NO_RAID_UNINTS++;
    }
    if ( $line =~ /^HORCM_CMD/ ) {
        $MARKER=1;
    }
  }
 close HORCM;

# Command output is like;
#PID  POLS U(%) SSCNT Available(MB) Capacity(MB)     Seq# Num LDEV# H(%)
#001  POLN   2     30      3195822      3293808     17353   2     5  70
#002  POLN   0      0       201932       201932     17353   1 12295  80
#007  POLN   0     20       198324       198324     17353  20 64768  70
 for my $RAID_UNIT (0..$NO_RAID_UNINTS-1) {
# Generate full command path
   my $CMD = $main::HORCM_CLIDIR . "raidvchkscan";
   my $CCICMD = get_cci_cmd($CMD);
   my $FULLCOMM = $CCICMD . " -v pid " . "$RAID_UNIT";

   my $POOL_USAGE_RESPONSE=qx($FULLCOMM 2>/dev/null);
   my @POOL_USAGE=split(/\n/,$POOL_USAGE_RESPONSE);
# discard the header line
   shift @POOL_USAGE;
#build a hash containing the usage for each pool
   foreach (@POOL_USAGE) {
     my ($SERIALNUMBER,$POOLID,$USAGE);
     if ( $_ =~ /(\d+)\s+\w+\s+(\d+)\s+\d+\s+\d+\s+\d+\s+(\d+)\s+\d+\s+\d+\s+\d+/ ) {
        ($POOLID,$USAGE,$SERIALNUMBER)=($1,$2,$3);
        $POOLID=sprintf("%d",$POOLID);
        $main::pool_usage_hash->{$SERIALNUMBER}->{$POOLID}=$USAGE;
     }
   }
 }
}

##########################################################################
# Match pair details
##########################################################################
sub match_pair_details {

 my $GROUP = shift|| exit_error("Match pair details error: Missing parameter","ERR_MATCH_NO_DETAIL");           # Device Group
 my $SIFT = shift|| exit_error("Match pair details error: Missing parameter","ERR_MATCH_NO_DETAIL");            # Search word
 my $MATCH = shift|| exit_error("Match pair details error: Missing parameter","ERR_MATCH_NO_DETAIL");           # Match word
 my $RUN_CLI = shift||0;                                                                                        # 0 = run pairdisplay, 1 = read global
 my @ALL_LINE = "";
 my $R_VAL = -1;                                        # Local return code variable

if ($RUN_CLI == 1 && @main::PAIRDISPLAY ne "") {
  @ALL_LINE = @main::PAIRDISPLAY;
} else {
# Generate full command path
    my $CMD = $main::HORCM_CLIDIR . "pairdisplay";
    my $CCICMD = get_cci_cmd($CMD);
    my $FULLCOMM = $CCICMD . " -g " . $GROUP . " -fxc -CLI";

    if ( $main::LMODE == 1 ) {
      $FULLCOMM =  $FULLCOMM . " -l";
    }

    # Execute the command
    @ALL_LINE = qx($FULLCOMM);
    $R_VAL = $?/256;
    if ( $R_VAL > 3 ) {
      log_message("Error: pairdisplay -l has now been sent", $main::LOGFILE);
      pair_display ($GROUP, 1);
      exit_error("Check pair display error: pairdisplay command generated an error: $FULLCOMM", $R_VAL, 1, 1);
    }
}

# Sift out the lines concerned
my @NO_TITLE = grep (/$SIFT/,@ALL_LINE);
my $total_number = @NO_TITLE;                           # Count the number of lines sifted out

# print "Number of lines ($SIFT) found: " . $total_number . "\n";

# Check to see if all these lines match
my @PAT_MATCH = grep (/$MATCH/,@NO_TITLE);
my $total_match = @PAT_MATCH;                           # Count the number of lines matched

# print "Number of matches ($MATCH) found: " . $total_match . "\n";

if ($total_number > 0 ) {
  if ($total_number == $total_match) {
    return 1;                                   # All found are matched
  } else {
      return 2;                                 # All found are not matched
    }
} else {
    return 0;                                   # None found
  }

}

##########################################################################
# Displays Journal Usage
##########################################################################
# Displays Journal Usage
sub journal_usage {

 my $GROUP = shift||exit_error("Pair display error: Missing parameter","ERR_PAIRDIS_NO_DET");           # Device Group;
# Generate full command path
 my $CMD = $main::HORCM_CLIDIR . "pairdisplay";
 my $CCICMD = get_cci_cmd($CMD);
 my $FULLCOMM = $CCICMD . " -g " . $GROUP ." -v jnl -l -CLI";

 my @JNL_USAGE = qx($FULLCOMM);
# remove the 1st line of the output
 shift @JNL_USAGE;
# usage % is column 5 of the output
 my @SPLIT=split(/\s+/,$JNL_USAGE[0]);
 $main::jnl_usage=$SPLIT[5];
 $main::jnl_group=sprintf("%d",$SPLIT[0]);
 return 0;

}

##########################################################################
sub set_horcminstance
{
 my $REPTYPE=shift;
 my $horcm_instance_hash={};
 # read the site config file into a hash
 open (SITE,"<$main::SITECONFIG");
 while ( my $line = <SITE> ) {
        if ( $line !~ /^#/ ) {
                chomp $line;
                #if ( $line =~ /^(\w+),(\w+),(\d+)$/ ) {
                #        my ($hostname,$reptype,$instance)=($1,$2,$3);
                my ($hostname,$reptype,$instance)=(split(/\,/,$line));
                        $horcm_instance_hash->{$hostname}->{$reptype}=$instance;
                #}
        }
 }
 close SITE;
 unless (defined $horcm_instance_hash->{$main::HOSTNAME}->{$REPTYPE}) {
                return 1;
 }
 return $horcm_instance_hash->{$main::HOSTNAME}->{$REPTYPE};

}


##########################################################################
sub set_env {

my $MRCF = shift || 0;                                          # Set the replication type, i.e. TrueCopy (=0) or ShadowImage (=1)
my $SPEED = shift|| 0;                                          # Determines which setting to change 1 = SPLIT, 2 = RESYNC , 3 = RESTORE, else = NONE
my $msg = "";

$ENV{HORCMINST} = $main::HORCMINST;                             # Set HORCM Instance

if ( $MRCF == 0 ) {
  delete $ENV{HORCC_MRCF};                                      # Set to TrueCopy
} else {
    $ENV{HORCC_MRCF} = 1;                                       # Set to ShadowImage
  }

return 0;

}

##########################################################################
sub check_HORCM
{
 $main::HORCM = "horcm$main::HORCMINST.conf";

 if( file_exists("$main::HORCM_DIR$main::HORCM") == 0 ) {
   log_message( "HORCM file is $main::HORCM" , $main::LOGFILE);

   # Check to see whether the HORCM Device Name is in the HORCM*.conf file
   if ( check_file("$main::HORCM_DIR$main::HORCM" , $main::DEVGROUP) == 0 ) {
     log_message( "HORCM Device Name is $main::DEVGROUP" , $main::LOGFILE);
   } else {
       log_message( "HORCM Device Name '$main::DEVGROUP' is not in the $main::HORCM file" , $main::LOGFILE);
       exit_error("HORCM Error: HORCM Device Name '$main::DEVGROUP' not defined in $main::HORCM file ","ERR_NO_HORCM_GROUP");
     }

 } else {
     log_message( "HORCM file '$main::HORCM_DIR$main::HORCM' does not exist" , $main::LOGFILE);
     exit_error("HORCM Error: HORCM file 'main::$main::HORCM_DIR$main::HORCM' does not exist","ERR_NO_HORCM_FILE");
   }

}

##########################################################################
# List all DEVICE GROUPS
##########################################################################
# list all the device groups and place them in an array
sub list_devgroups
{
my @DEV_GROUPS=();
# set a marker to 0
my $horcmfile= $main::HORCM_DIR . "horcm" . $main::HORCMINST . ".conf";
my $MARKER=0;
# read in the horcm config file;
 open (HORCM,"<$horcmfile");
 while ( my $line = <HORCM> ) {
    chomp $line;
    if (( $MARKER == 1 ) && ( $line !~ /^#/ )) {
        my @SPLIT=split(/\s/,$line);
        push @DEV_GROUPS,$SPLIT[0];
    }
    if ( $line =~ /^HORCM_INST/ ) {
        $MARKER=1;
    }
  }
 close HORCM;
 return @DEV_GROUPS;

}
##########################################################################
sub check_location
{

 my $MSG = "";
 my $R_VAL = -1;

 if (match_pair_details($main::DEVGROUP,"P-VOL", " L ", 1) == 1 ) {
   $MSG = "Primary (P-VOL) volumes are at local site";
   $R_VAL = 0;
 } elsif (match_pair_details($main::DEVGROUP,"S-VOL", " L ", 1) == 1 ) {
     $MSG = "Secondary (S-VOL) volumes are at local site";
   $R_VAL = 1;
   } elsif (match_pair_details($main::DEVGROUP,"SMPL", " - ", 1) == 1 ) {
       $MSG = "Volumes in Simplex (SMPL) mode";
       $R_VAL = 2;
     } else {
       $MSG = "Unable to establish location of volumes";
       exit_error("Site Error: $MSG","ERR_SITE_NOT_KNOWN");
     }

 log_message($MSG, $main::LOGFILE);

 return $R_VAL;
}

##########################################################################
sub exit_error
{
 my $ERR_MSG = shift || "Unknown Error";                # Perl application error message
 my $ERR_RET = shift || "Unknown";                              # Return Code. Either a description or a number
 my $LOOKUP = shift || 0;                               # 0 = normal lookup, 1 = reverse lookup
 my $ERR_LIST = shift || 0;                             # 0 = Perl application error listings, 1 = HDS error listings
 my $ERR_RET2 = 0;                                      # The value to look up. Either a description or a number
 my $ERR_CODE = -1;                             # Default error value at start of program
 my $ERR_DESC = "START";                        # Default error decription at start of program


 if ( $ERR_LIST == 0 ) {                                # Perl Application error listings
   $ERR_RET2 = errorlist($ERR_RET, $LOOKUP);
 } else {                                                       # HDS error listings
     $ERR_RET2 = hds_errorlist($ERR_RET, $LOOKUP);
   }

 if ( $LOOKUP == 0 ) {                          # Passed an error description to retrieve an error code number
   $ERR_DESC = $ERR_RET;
   $ERR_CODE = $ERR_RET2;
 } else {                                       # Passed an error code number to retrieve an error description
     $ERR_DESC = $ERR_RET2;
     $ERR_CODE = $ERR_RET;
   }

 $ERR_MSG = "Error Code: $ERR_DESC ($ERR_CODE). $ERR_MSG";
 log_message($ERR_MSG, $main::ERRFILE);
 log_message("Program $main::FULLNAME terminated due to error $ERR_DESC ($ERR_CODE).",$main::ERRFILE);
 log_message("See error log file '$main::ERRFILE'.", $main::LOGFILE);

 clean_up;

 exit $ERR_CODE;
}

##########################################################################
# Clean-up
##########################################################################
# Clean up before exitting
sub clean_up
{

 # Print trailing new lines at end of logs
 log_message("Script ended", $main::LOGFILE);

# Close logfile if still open
 if ($main::LOGFILE) {
   close $main::LOGFILE;
 }


return 0;

}


##########################################################################
sub execute_com
{
 my $FULLCOMM = shift||die;
 my $RET_STATE = shift || "";
 my $HORC_RET = shift || 0;                                      # Set to 1 is for horctakeover commands which return values 1 to 5
 my $R_VAL = -1;
 my $TMODE=0;                                                    # Time out mode. 0->wait until done, do not time out
 my $msg;


 log_message("Command '$FULLCOMM' executing...", $main::LOGFILE);
 print qx($FULLCOMM);                       
 $R_VAL = $?/256;
 if ( 1 && $R_VAL == 225) {
   $msg = "Program returned $R_VAL (" . hds_errorlist($R_VAL, 1) . ")";
   log_message($msg, $main::LOGFILE);
   $R_VAL = 0;
   pair_display ($main::DEVGROUP,1);
  } else {
    if ( $HORC_RET == 0 ) {                                # Check if a horctakeover command
      if ( $R_VAL != 0  ) {
        exit_error("HDS Command Error: Command '$FULLCOMM' returned an error",$R_VAL, 1, 1);
      }
      else {
        log_message("Command returned $R_VAL", $main::LOGFILE);
      }
    } else {
      if ( $R_VAL == 1 ) {
        log_message("Swap-takeover was successfully executed.",  $main::LOGFILE);
      }
      elsif ( $R_VAL == 2 ) {
	log_message("SVOL-takeover was successfully executed.", $main::LOGFILE);
      }
      elsif ( $R_VAL == 3 ) {
	log_message("PVOL-SMPL-takeover was successfully executed.", $main::LOGFILE);
      }
      elsif ( $R_VAL == 4 ) {
	log_message("PVOL-PSUE-takeover was successfully executed.", $main::LOGFILE);
      }
      elsif ( $R_VAL == 5 ) {
	log_message("SVOL-SSUS-takeover was successfully executed.", $main::LOGFILE);
      }
      elsif ( $R_VAL > 5  ) {
        exit_error("HDS Command Error: Command '$FULLCOMM' returned an error",$R_VAL, 1, 1);
      }
    }
  }


  # Wait till command completes, if required
  if ( $RET_STATE ne "" ) {
     log_message("Waiting for pair state to change to '$RET_STATE'...", $main::LOGFILE);
    if (wait_pair_status($main::DEVGROUP, $RET_STATE , $TMODE) == 0) {
      log_message("'$FULLCOMM' complete", $main::LOGFILE);
    }
  }
return $R_VAL;
}

##########################################################################
# Lock the Device Group
##########################################################################
# Lock the Device Group
sub lock_devgroup
{
 my $LOCKDIR="../etc/";
 my $LOCKFILE=$LOCKDIR . $main::DEVGROUP . ".lck";
 my $DEVGROUP_LOCK=1;
 open (LOCK,">$LOCKFILE");

# Aquire lock on devgroup
 while ($DEVGROUP_LOCK) {
        if (flock LOCK, 2 | 4) {
                log_message("DEVGROUP $main::DEVGROUP has been locked",$main::LOGFILE);
                $DEVGROUP_LOCK=0;
        }
        else {
                warn "DEVGROUP $main::DEVGROUP is already locked; waiting...\n";
                sleep 20;
        }
 }
}

##########################################################################
# Unlock the Device Group
##########################################################################
# Unlock the Device Group
sub unlock_devgroup
{
 my $LOCKDIR="../etc/";
 my $LOCKFILE=$LOCKDIR . $main::DEVGROUP . ".lck";
 open (LOCK,">$LOCKFILE");
 flock LOCK,8;
 log_message("DEVGROUP $main::DEVGROUP has been unlocked",$main::LOGFILE);
 close LOCK;
}

###########################################################################
# End of Script
###########################################################################
1