#!/opt/perl/5.8.0/bin/perl -w

use strict;

my $PBRUN="cat /dev/null | pbrun";
if (defined $ENV{"PBRUN"} )
{
    $PBRUN=$ENV{"PBRUN"}
}


##########################################################################
#
# script resume_async
#
# C.A. Clayton 	4th July 2005 	original version V1.0
#		
# Invocation:	resume_async <DGName> <RDF group> 
#
# Arguments: 	<DGName>	- Device Group Name
#               <RDF group> 	- RDF group number of async link
#
# Example:	resume_async App1 4
#
#
# Return codes:
#		0 	Success
#		1	Wrong number of arguments supplied
#		2 	A symmir verify command showed system not in right state
#		3 	A symmir command failed to run correctly
#
# Assumptions:
#
# 1) PIT BCVs are defined in the device group
#
# 2) The PIT BCV is established. Note this script will not run
#    if BCV is not established or establishing as this may 
#    may indicate that the SRDF/A outage occurred during a 
#    previously incomplete resunc operation and hence the BCV
#    should NOT be established against a potentially inconsistent
#    R2. 
#
# 
# 1) Verify remote BCV is fully established
# 2) Split remote BCV
# 3) Wait until finished splitting in the backgroup
# 4) Disable consistency.
# 5) Set mode to adaptive copy. (ACP_Disk)
# 6) Establish RDF
# 7) Wait until RDF is down to <30,000 tracks invalid.
# 8) Set mode to async mode 
# 9) Wait until all tracks sync'ed.
# 10) Enable consistency
# 11) Establish remote BCV
# 12) Wait until remote BCV fully established before exiting

##########################################################################
#
# Variables used:
#
my $dgname;		# Device Group Name
#
my $rdf_group;		# Current RDF group being processed
#
my $output;		# Output returned from external commands
#
my $returncode;		# Return code from external commands
#
my $steparg;		# Step # to continue running at.
#
my $statusref;		# Hash reference returned from checkstatuses routine.
#
my %stepreq;		# Step requirements - what SRDF/BCV should be for this step.
#
##########################################################################
#
# Parse input parameters. Exactly 2 arguments expected i.e. $#ARGV = 1
#

# If passed 3 arguments, then looking for special handling.
# 3rd argument should be which step to jump to. No 3rd argument means 
# run like normal, 3rd argument=99 means pick next step automatically.
if ( $#ARGV == 2 ) {
   $steparg = $ARGV[2];
} elsif ( $#ARGV != 1 ) {
   print "Incorrect number of arguments for script resume_async\n";
   print "USAGE: resume_async <DGName> <RDF group>\n";
   exit(1);
}
$dgname = $ARGV[0];
$rdf_group = $ARGV[1];


##########################################################################
#
# 1) Check remote BCV is synced up first
#
undef %stepreq;
$stepreq{"step"}=1;
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;
    
    print "\n*** Starting SRDF/A resynchronization exercise\n";
    print "*** Step 1: Verifying remote BCVs already synced up\n";

    $statusref=checkstatuses("BCV");
    if ($statusref->{"BCV"}{"Synchronized"} != $statusref->{"BCV"}{"Total"})
    {
	print "PIT BCV(s) not already fully synced up. Script exiting.\n";
	exit (2)
    }
} else {
    print "Skipping step 1\n";
}


##########################################################################
#
# 2) Split remote BCV
#
undef %stepreq;
$stepreq{"step"}=2;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Synchronized == Total count of BCVs.
# and SRDF Consistency Enabled == 1,
# and SRDF Asynchronous devs == Total devs in this mode,
# and SRDF Consistent devs == 0.
@{$stepreq{"BCV"}}=(">>Synchronized<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 1",
		     ">>Asynchronous<< == >>ModeTotal<<",
		     ">>Consistent<< == 0");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;
    
    print "*** Step 2: Splitting remote BCVs\n";
    $output=`${PBRUN} /usr/symcli/bin/symmir -g $dgname -consistent -rdf split -noprompt`;
    $returncode = ($? >> 8);
    
    if ( $returncode != 0 ) {
       print "PIT BCV(s) failed to split. Script exiting.\n";
       exit (3)
    }
} else {
    print "Skipping step 2\n";
}

##########################################################################
#
# 3) Wait until finished splitting in the background
#
undef %stepreq;
$stepreq{"step"}=3;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires that BCV count Synchronized == 0.
# and SRDF Consistency Enabled == 1,
# and SRDF Asynchronous devs == Total devs in this mode,
# and SRDF Consistent devs == 0.
@{$stepreq{"BCV"}}=(">>Synchronized<< == 0");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 1",
		     ">>Asynchronous<< == >>ModeTotal<<",
		     ">>Consistent<< == 0");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;
    
    print "*** Step 3: Waiting for background BCV split for $dgname to finish\n";
    printf(" BCVs   BG BCVs   Total\n");
    printf("Split  Splitting   BCVs\n");
    $statusref=checkstatuses("BCV");
    while ( ($statusref->{"BCV"}{"Split"} != $statusref->{"BCV"}{"Total"}) ||
	    ($statusref->{"BCV"}{"SplitBG"} != 0) )
    {
	printf("%5d  %8d    %5d                    \r",
	       $statusref->{"BCV"}{"Split"},
	       $statusref->{"BCV"}{"SplitBG"},
	       $statusref->{"BCV"}{"Total"} );
	sleep 5;
	$statusref=checkstatuses("BCV");
    }
    printf("%5d  %8d    %5d                    \n",
	   $statusref->{"BCV"}{"Split"},
	   $statusref->{"BCV"}{"SplitBG"},
	   $statusref->{"BCV"}{"Total"} );
} else {
    print "Skipping step 3\n";
}

##########################################################################
#
# 4) Disable Consistency.
#
undef %stepreq;
$stepreq{"step"}=4;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Enabled == 1.
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 1");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;
    
    print "*** Step 4: Setting consistency disabled for $dgname\n";
    $output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname -rdfg $rdf_group disable -nop`;
    $returncode = ($? >> 8);

    #
    # Return code from verify should be 0
    #
    if ( $returncode != 0 ) {
       print "Mode change to adapative copy failed. Script exiting.\n";
       exit(2);
    }
} else {
    print "Skipping step 4\n";
}

##########################################################################
#
# 5) Setting mode to adaptive copy. (ACP_Disk)
#
undef %stepreq;
$stepreq{"step"}=5;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Disabled (Enabled == 0)
# and SRDF count Async == Total
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 0",
		     ">>Asynchronous<< == >>ModeTotal<<");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;

    print "*** Step 5: Setting mode for $dgname to adaptive copy\n";
    $output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname -rdfg $rdf_group set mode acp_disk -nop`;
    $returncode = ($? >> 8);

    #
    # Return code from verify should be 0
    #
    if ( $returncode != 0 ) {
       print "Mode change to adapative copy failed. Script exiting.\n";
       exit(2);
    }
} else {
    print "Skipping step 5\n";
}

##########################################################################
#
# 6) Establish RDF
#
undef %stepreq;
$stepreq{"step"}=6;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Disabled (Enabled == 0)
# and SRDF count ACP_Disk == Total
# and SRDF count Split == Total or Suspended == Total
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 0",
		     ">>Adaptive Copy Disk<< == >>ModeTotal<<",
		     "(>>Split<< == >>StateTotal<<) || (>>Suspended<< == >>StateTotal<<)");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;

    print "*** Step 6: Establish RDF link for $dgname\n";
    $output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname -rdfg $rdf_group establish -nop`;
    $returncode = ($? >> 8);
    
    #
    # Return code from verify should be 0
    #
    if ( $returncode != 0 ) {
       print "Incremental establish operation failed. Script exiting.\n";
       exit(2);
    }
} else {
    print "Skipping step 6\n";
}

##########################################################################
#
# 7) Wait until RDF is down to < 30,000 tracks invalid.
#
undef %stepreq;
$stepreq{"step"}=7;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Disabled (Enabled == 0)
# and SRDF count ACP_Disk == Total
# and SRDF count Synchronized + SyncInProg == Total.
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 0",
		     ">>Adaptive Copy Disk<< == >>ModeTotal<<",
		     "(>>Synchronized<< + >>SyncInProg<<) == >>StateTotal<<");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;

    my $lasttracks=0;
    my $lasttime;
    
    print "*** Step 7: Waiting until $dgname RDF is down to <30,000 tracks invalid\n";
    printf("  R2s     R2s       Invalid    Est. Time\n");
    printf(" Sync'd SyncInProg  Tracks     Remaining\n");
    $statusref=checkstatuses("SRDF");
    while ( ($statusref->{"SRDF"}{"Synchronized"} != $statusref->{"SRDF"}{"StateTotal"}) &&
	    ($statusref->{"SRDF"}{"TotalTracks"} > 30000) )
    {
	my $thistime=time;
	my $thistracks=$statusref->{"SRDF"}{"TotalTracks"};
	if ($thistracks >= $lasttracks) 
	{
	    undef $lasttime;
	}
	if ($lasttime)
	{
	    my $lastcycle=$thistime-$lasttime;
	    my $trackslastcycle=$lasttracks-$thistracks;
	    my $trackspersec=$trackslastcycle/$lastcycle;
	    my $secsremaining=$thistracks/$trackspersec;
	    printf(" %5d   %5d    %9d     %02d:%02d:%02d        \r",
		   $statusref->{"SRDF"}{"Synchronized"},
		   $statusref->{"SRDF"}{"SyncInProg"},
		   $statusref->{"SRDF"}{"TotalTracks"},
		   (gmtime $secsremaining)[2,1,0]);
	} else {
	    printf(" %5d   %5d    %9d                           \r",
		   $statusref->{"SRDF"}{"Synchronized"},
		   $statusref->{"SRDF"}{"SyncInProg"},
		   $statusref->{"SRDF"}{"TotalTracks"} );
	}
	$lasttime=$thistime;
	$lasttracks=$thistracks;
	sleep 5;
	$statusref=checkstatuses("SRDF");
    }
	printf(" %5d   %5d    %9d                            \n",
	      $statusref->{"SRDF"}{"Synchronized"},
	      $statusref->{"SRDF"}{"SyncInProg"},
	      $statusref->{"SRDF"}{"TotalTracks"} );
} else {
    print "Skipping step 7\n";
}
    
##########################################################################
#
# 8) Set mode to async mode
#
undef %stepreq;
$stepreq{"step"}=8;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Disabled (Enabled == 0)
# and SRDF count ACP_Disk == Total
# and SRDF count Synchronized + SyncInProg == Total.
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 0",
		     ">>Adaptive Copy Disk<< == >>ModeTotal<<",
		     "(>>Synchronized<< + >>SyncInProg<<) == >>StateTotal<<",
		     "(>>TotalTracks<< < 30000)" );
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;

    print "*** Step 8: Change mode back to async for $dgname\n";
    $output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname -rdfg $rdf_group set mode async -force -nop`;
    $returncode = ($? >> 8);

    #
    # Return code from verify should be 0
    #
    if ( $returncode != 0 ) {
       print "Mode change to async failed. Script exiting.\n";
       exit(2);
    }    
} else {
    print "Skipping step 8\n";
}


##########################################################################
#
# 9) Wait for tracks to completely catch up.
#
undef %stepreq;
$stepreq{"step"}=9;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Disabled (Enabled == 0)
# and SRDF count Async == Total
# and SRDF count Synchronized + SyncInProg + Consistent == Total.
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 0",
		     ">>Asynchronous<< == >>ModeTotal<<",
		     "(>>Synchronized<< + >>SyncInProg<< + >>Consistent<<) == >>StateTotal<<");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;

    my $lasttracks=0;
    my $lasttime;

    print "*** Step 9: Waiting until $dgname RDF is finished syncing\n";
    printf("  R2s     R2s         R2s    Invalid    Est. Time\n");
    printf(" Sync'd SyncInProg  Consist  Tracks     Remaining\n");
    $statusref=checkstatuses("SRDF");
    while ($statusref->{"SRDF"}{"Consistent"} != $statusref->{"SRDF"}{"StateTotal"})
    {
	my $thistime=time;
	my $thistracks=$statusref->{"SRDF"}{"TotalTracks"};
	if ($thistracks >= $lasttracks)
	{
	    undef $lasttime;
	}
	if ($lasttime)
	{
	    my $lastcycle=$thistime-$lasttime;
	    my $trackslastcycle=$lasttracks-$thistracks;
	    my $trackspersec=$trackslastcycle/$lastcycle;
	    my $secsremaining=$thistracks/$trackspersec;
	    printf(" %5d   %5d     %5d   %9d    %02d:%02d:%02d          \r",
		   $statusref->{"SRDF"}{"Synchronized"},
		   $statusref->{"SRDF"}{"SyncInProg"},
		   $statusref->{"SRDF"}{"Consistent"},
		   $statusref->{"SRDF"}{"TotalTracks"},
		   (gmtime $secsremaining)[2,1,0]);
	} else {
	    printf(" %5d   %5d     %5d   %9d                            \r",
		   $statusref->{"SRDF"}{"Synchronized"},
		   $statusref->{"SRDF"}{"SyncInProg"},
		   $statusref->{"SRDF"}{"Consistent"},
		   $statusref->{"SRDF"}{"TotalTracks"} );
	}
	$lasttime=$thistime;
	$lasttracks=$thistracks;
	sleep 5;
	$statusref=checkstatuses("SRDF");
    }
    printf(" %5d   %5d     %5d   %9d                        \n",
	   $statusref->{"SRDF"}{"Synchronized"},
	   $statusref->{"SRDF"}{"SyncInProg"},
	   $statusref->{"SRDF"}{"Consistent"},
	   $statusref->{"SRDF"}{"TotalTracks"} );
} else {
    print "Skipping step 9\n";
}


##########################################################################
#
# 10) Enable consistency
#
undef %stepreq;
$stepreq{"step"}=10;
undef $stepreq{"check"}; # Check both BCV and SRDF.
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Disabled (Enabled == 0)
# and SRDF count Async == Total
# and SRDF count Consistent == Total.
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 0",
		     ">>Asynchronous<< == >>ModeTotal<<",
		     ">>Consistent<< == >>StateTotal<<");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;
    
    print "*** Step 10: Enabling consistency for $dgname\n";
    $output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname -rdfg $rdf_group enable -nop`;
    $returncode = ($? >> 8);
    #
    #
    # Return code from verify should be 0
    #
    if ( $returncode != 0 ) {
       print "Mode change to async failed. Script exiting.\n";
       exit(2);
    }
} else {
    print "Skipping step 10\n";
}

##########################################################################
#
# 11) Establish remote BCV
#
undef %stepreq;
$stepreq{"step"}=11;
# Step requires BCV count of Split == Total
# and BCV count of SplitBG == 0
# and SRDF Consistency Enabled == 1
# and SRDF count Async == Total
# and SRDF count Consistent == Total.
@{$stepreq{"BCV"}}=(">>SplitBG<< == 0",
		 ">>Split<< == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 1",
		     ">>Asynchronous<< == >>ModeTotal<<",
		     ">>Consistent<< == >>StateTotal<<");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;
    
    print "*** Step 11: Incrementally establish remote BCV for $dgname\n";
    $output=`${PBRUN} /usr/symcli/bin/symmir -g $dgname establish -rdf -force -nop`;
    $returncode = ($? >> 8);

    #
    # Return code from verify should be 0
    #
    if ( $returncode != 0 ) {
       print "Incremental etablish operation failed. Script exiting.\n";
       exit(2);
    }
} else {
    print "Skipping step 11\n";
}

##########################################################################
#
# 12) Wait until remote BCV fully established before exiting
#
undef %stepreq;
$stepreq{"step"}=12;
# Step requires BCV count of Synchronized + SyncInProg == Total
# and SRDF Consistency Enabled == 1
# and SRDF count Async == Total
# and SRDF count Consistent == Total.
@{$stepreq{"BCV"}}=("(>>Synchronized<< + >>SyncInProg<<) == >>Total<<");
@{$stepreq{"SRDF"}}=(">>Enabled<< == 1",
		     ">>Asynchronous<< == >>ModeTotal<<",
		     ">>Consistent<< == >>StateTotal<<");
if (checksteprequirements(\%stepreq,$steparg) == 0)
{
    undef $steparg;

    my $lasttracks=0;
    my $lasttime;

    print "***  Step 12: Wait until remote BCV fully established for $dgname\n";
    printf("  BCVs     BCVs     Invalid    Est. Time\n");
    printf(" Sync'd SyncInProg   Tracks    Remaining\n");
    $statusref=checkstatuses("BCV");
    while ($statusref->{"BCV"}{"Synchronized"} != $statusref->{"BCV"}{"Total"})
    {
	my $thistime=time;
	my $thistracks=$statusref->{"BCV"}{"TotalTracks"};
	if ($thistracks >= $lasttracks)
	{
	    undef $lasttime;
	}
	if ($lasttime)
	{
	    my $lastcycle=$thistime-$lasttime;
	    my $trackslastcycle=$lasttracks-$thistracks;
	    my $trackspersec=$trackslastcycle/$lastcycle;
	    my $secsremaining=$thistracks/$trackspersec;
	    printf(" %5d   %5d    %9d    %02d:%02d:%02d          \r",
		   $statusref->{"BCV"}{"Synchronized"},
		   $statusref->{"BCV"}{"SyncInProg"},
		   $statusref->{"BCV"}{"TotalTracks"},
		   (gmtime $secsremaining)[2,1,0]);
	} else {
	    printf(" %5d   %5d    %9d                      \r",
		   $statusref->{"BCV"}{"Synchronized"},
		   $statusref->{"BCV"}{"SyncInProg"},
		   $statusref->{"BCV"}{"TotalTracks"});
	}
	$lasttime=$thistime;
	$lasttracks=$thistracks;
	sleep 5;
	$statusref=checkstatuses("BCV");
    }
    printf(" %5d   %5d   %9d                      \n",
	   $statusref->{"BCV"}{"Synchronized"},
	   $statusref->{"BCV"}{"SyncInProg"},
	   $statusref->{"BCV"}{"TotalTracks"} );
} else {
    print "Skipping step 12.\n Check step argument - provided ($steparg) argument invalid.\n";
    exit (1);
}
    

print "\n***  SRDF/A Resynchronization exercise for device group $dgname completed \n";
exit (0);



#
# Check RDF and BCV status
#
sub checkstatuses
{
    # Get statuses of SRDF and/or BCVs as indicated by argument called with.
    # If only BCV status needed - call with "BCV" argument. If only SRDF status
    # needed call with "SRDF". As appropriate, get output from
    # symrdf verify -enabled -summary and/or  symmir verify -summary.
    # Go through and check counts and use that to identify state of RDF/BCV groups.
    # (Better than dozens of symrdf/symmir verify -state commands.)

    # First go through symrdf output, (if required) and itemize states in variables.
    # Then go through variables, look for states invalid for us here, or inconsistencies
    # that don't look right.
    # Then go through symmir output is required.
    # Then go through symmir variables, looking for states and consistencies.
    # Once we know everything looks "OK", then return the states in a hash to the caller.

    # If "BCV" is passed as an argument, just check BCVs. If "SRDF" is passed, just SRDF.
    my ($check) = @_;

    my %returnhash; # The hash we'll be returning a reference to.

    # Work variables needed for this routine.
    my $output; # Output from the called solutions enabler command.
    my $returncode; # The return code from the solutions enabler command.
    my $space; # Holder for null/space building text keys.
    my $type; # "Type" of state/mode - used for keys into hash.
    my $count; # Count of devs.
    my $RDFstate=""; # The state of the RDF group.
    my $RDFmode=""; # The mode of the RDF group.
    my $BCVstate=""; # The state of the BCVs.
    my @f; # Fields split out of text;
    my $tracks; # Number of invalid tracks.
    my $mb; # Invalid MBs.
    
    if ((! defined $check) || ($check ne "BCV"))
    {
    # First go through symrdf output.
	$output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname verify -enabled -summary 2>&1`;
	$returncode = ($? >> 8);
	if ($returncode == 1) {
	    print "symrdf verify -enabled -summary returned RC 1.\n";
	    print $output;
	    exit(2);
	}
	my $statedelim=""; # Holder for null/text as delimiter building $RDFstate;
	my $modedelim=""; # Holder for null/text as delimiter building $RDFmode;
	for (split /^/, $output) {
	    if (/^\s*(Consistent|Synchronized|SyncInProg|Suspend|Split|Failed|R1 Upd|TransIdle|Partition|Invalid)/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<$#f;$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$count=$f[$#f];
		$returnhash{"SRDF"}{$type}=$count;
		if ($count)
		{
		    $RDFstate="$RDFstate$statedelim$type";
		    $statedelim="/";
		}
	    }
	    if ((/^\s*Total\s*[0-9]+\s*$/)&&(! defined $returnhash{"SRDF"}{"StateTotal"})) {
		($type,$returnhash{"SRDF"}{"StateTotal"})=split;
	    }
	    if (/^\s*(Synchronous|Semi-Sync|Asynchronous|Adaptive Copy)/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<$#f;$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$count=$f[$#f];
		$returnhash{"SRDF"}{$type}=$count;
		if ($count)
		{
		    $RDFmode="$RDFmode$modedelim$type";
		    $modedelim="/";
		}
	    }
	    if ((/^\s*Total\s*[0-9]+\s*$/)&&(defined $returnhash{"SRDF"}{"StateTotal"})) {
		($type,$returnhash{"SRDF"}{"ModeTotal"})=split;
		
		if ($returnhash{"SRDF"}{"StateTotal"} != $returnhash{"SRDF"}{"ModeTotal"})
		{
		    print "Inconsistent symrdf counts. Totals not equal.\n";
		    print "$output";
		    exit(2);
		}
	    }
	    if (/^\s*Total.*Invalid/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<($#f-1);$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$tracks=$f[$#f-1];
		$returnhash{"SRDF"}{"TotalTracks"}+=$tracks;
		$mb=$f[$#f];
		$returnhash{"SRDF"}{"TotalMB"}+=$mb;
		$returnhash{"SRDF"}{$type}{"Tracks"}=$tracks;
		$returnhash{"SRDF"}{$type}{"MB"}=$mb;
	    }
	}
	
	# If symrdf verify -enabled returns 0, then it is enabled.
	if ($returncode == 0)
	{
	    $returnhash{"SRDF"}{"Enabled"}=1;
	} else {
	    $returnhash{"SRDF"}{"Enabled"}=0;
	}

	# We have the SRDF State, now see what doesn't look right.
	
	# Resume will only deal with Suspends or Splits.
	if ($RDFstate =~ 
	    /Failed Over|R1 Updated|R1UpdInProg|TransIdle|Partitioned|Invalid/)
	{
	    print "SRDF status not appropriate for resumption from suspend.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}
	
	# Synchronous or Semi-Synchronous.
	if ($RDFmode =~ /Synchronous/)
	{
	    print "SRDF IN SYNC MODE. CORRECT ASAP.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}
	
	# Some Consistent and some Sync(ing) or Suspended or Split. That shouldn't be...
	if (($RDFstate =~ /Consistent/) &&
	    ($RDFstate =~ /Sync|Suspended|Split/)) 
	{
	    print "SRDF State seems inconsistent. (consistent+) Please check.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}
	
	# Some Sync(ing) and some Suspended or Split. That shouldn't be either...
	if (($RDFstate =~ /Sync/) &&
	    ($RDFstate =~ /Suspended|Split/))
	{
	    print "SRDF State seems inconsistent. (Sync/Suspend/Split) Please check.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}

	# Some Suspended and some Split. That shouldn't be either...
	if (($RDFstate =~ /Suspend/) &&
	    ($RDFstate =~ /Split/))
	{
	    print "SRDF State seems inconsistent. (Suspend/Split) Please check.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}
	
	# Some Async and some Adaptive Copy. That also shouldn't be...
	if (($RDFmode =~ /Asynchronous/) &&
	    ($RDFmode =~ /Adaptive Copy/))
	{
	    print "SRDF Mode seems inconsistent. (async/acp) Please check.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}
	
	# Consistency Enabled should only be when in async mode.
	if (($RDFmode =~ /Enabled/) &&
	    ($RDFmode =~ /Adaptive Copy/))
	{
	    print "SRDF Mode seems incorrect. (Enabled/acp) Please check.\n";
	    print "Status: $RDFstate\n";
	    print "Mode: $RDFmode\n";
	    print "$output";
	    exit(2);
	}
    } #if check eq BCV
    #Now do the same thing for the BCVs.

    if ((! defined $check) || ($check ne "SRDF"))
    {
	$output=`${PBRUN} /usr/symcli/bin/symmir -rdf -g $dgname verify -summary 2>&1`;
	$returncode = ($? >> 8);
	if ($returncode == 1) {
	    print "symmir verify -summary returned RC 1.\n";
	    print $output;
	    exit(2);
	}
	my $statedelim=""; # Holder for null/text as delimiter building $BCVstate;
	for (split /^/, $output) {
	    if (/^\s*(Sync|Rest|Split|Invalid|Total[^A-Za-z]+[0-9])/)
	    {
		my $bg1; # Text holders for BG split status row.
		my $bg2;
		($type,$count,$bg1,$bg2)=split;
		$returnhash{"BCV"}{$type}=$count;
		if (($count)&&(!/Total/))
		{
		    $BCVstate="$BCVstate$statedelim$type";
		    $statedelim="/";
		}
		if (/Split.*bg/)
		{
		    ($returnhash{"BCV"}{"SplitBG"})=split /\]/, $bg2;
		    if ($returnhash{"BCV"}{"SplitBG"})
		    {
			$BCVstate="$BCVstate${statedelim}SplitBG";
			$statedelim="/";
		    }
		} # if Split.*bg
	    } # if Sync|Rest|Split...
	    if (/^\s*Total.*Invalid/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<($#f-1);$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$tracks=$f[$#f-1];
		$returnhash{"BCV"}{"TotalTracks"}+=$tracks;
		$mb=$f[$#f];
		$returnhash{"BCV"}{"TotalMB"}+=$mb;
		$returnhash{"BCV"}{$type}{"Tracks"}=$tracks;
		$returnhash{"BCV"}{$type}{"MB"}=$mb;
	    }
	} # for split...
    } # if check eq SRDF
    
    return (\%returnhash);
    
} # sub checkstatuses

####
# checksteprequirements
#
# Take a hash reference to the step requirements and the step arg as parameters.
#
# Perform SRDF/BCV checks against the requirements. 
# If OK to perform this step return 0.
# If not the right step, return 1. 
# If right step, but the requirements don't match, Exit.
#
# If $steparg is undefined, then return 0 - this means prior checks have been run
# and means we're just continuing on.

sub checksteprequirements
{
    my ($reqref,$steparg) = @_;
    my $ret = 0;

    if (! defined $steparg)
    {
	return 0;
    }

    if ($reqref->{"step"} != $steparg)
    {
	return 1;
    }

    print "Checking status for step $steparg...\n";

    my $checkstatusref=checkstatuses($reqref->{"check"});
    
    if (defined $reqref->{"SRDF"})
    {
	my $i;
	foreach $i (@{$reqref->{"SRDF"}})
	{
	    my $ex = $i;
	    $ex =~ s/\>\>([^\<]+)\<\</\$checkstatusref->{\"SRDF\"}{\"$1\"}/g;
	    if (! eval $ex)
	    {
		$ret |= 2;
		die "SRDF Status not correct for step $steparg. ($i)\n";
	    }
	}
    }
    if (defined $reqref->{"BCV"})
    {
	my $i;
	foreach $i (@{$reqref->{"BCV"}})
	{
	    my $ex = $i;
	    $ex =~ s/\>\>([^\<]+)\<\</\$checkstatusref->{\"BCV\"}{\"$1\"}/g;
	    if (! eval $ex)
	    {
		$ret |= 2;
		die "BCV Status not correct for step $steparg. ($i)\n";
	    }
	}
    }
    return $ret;
}