#!/usr/bin/perl -w

#
# 20130412 - changed interpreter path to /usr/bin/perl to allow use on
#     Solaris and Linux workstations.
#   Also remove absolute paths to SE commands so PBRUN doesn't break.
# 20130926 - Fixed symrdf verify command with required -rdfg switch.
#   Also added emphasis to error that finds Sync mode RDF groups.
# 20131105 - Added version variable and reporting/logging of version
#     for future debugging assistance.
# 20131106 - Adding additional error checking for SE command execution
#     including catching of states other than just "FAIL".
# 20131125 - Changing PBRUN defaults - pbrun on Solaris and nothing on Linux.
# 20140728 - Captures ticket for the script
#
# 20150910 - Tear apart resume_srdfa and turn into a new failback_srdfa

# PLEASE READ THIS:
#
# This is the same code as resume_srdfa (as of 9/10/2015) modified to perform
# Failback steps instead of split/suspend resumes. 
# The resume steps were truncated (doesn't need to worry about enable or srdf/a
# mode for example). Otherwise, it is very similar. BCV split/establish is the
# same. Instead of SRDF establish it runs SRDF failback. Because the steps are
# similar, I have left the step numbers the same - just removing the unneeded
# steps in between. This is why the step numbers jump.
#
# Step condition checks were modified so the status of the BCVs/SRDF group should
# be correct for a failover status. That said - I HAVE NOT TESTED THIS!
#
# The progam has been made safe in that it will terminate after the first step it
# runs. If not given a step #, it will run step 1 and stop. If given a different
# step, it will go to that step and then stop. This allows the program to be tested
# by going to the steps manually and checking the results of checkstatuses and the
# defined step requirements. 
#
# If the program tests fine - then the labeled sections to the tests can be removed.
# If there are issues - the step requirements initialized at the end of the
# program will need to be corrected.
#


my $version="20150910";

use strict;
use POSIX;
# Carp::longmess is used by cluck to give more detailed info about warnings.
# Binding __WARN__ to log these warnings.
use warnings;
use Carp ();
local $SIG{__WARN__} = sub { 
    my $msg = shift; 
    my $lm = Carp::longmess $msg; 
    warn $lm; 
    print OUT $lm;
};

my %PBdefaults=(
    SunOS => "cat /dev/null | pbrun",
    Linux => ""
    );
my $os=(uname)[0];
my $PBRUN=$PBdefaults{$os};
if (defined $ENV{"PBRUN"} )
{
    $PBRUN=$ENV{"PBRUN"}
}


##########################################################################
#
# script failback_async
#
# C.A. Clayton 	4th July 2005 	original version V1.0
#		
# Invocation:	failback_async <DGName> <RDF group> 
#
# Arguments: 	<DGName>	- Device Group Name
#               <RDF group> 	- RDF group number of async link
#
# Example:	failback_async App1 4
#
#
# Return codes:
#		0 	Success
#		1	Wrong number of arguments supplied
#		2 	A symmir verify command showed system not in right state
#		3 	A symmir command failed to run correctly
#
# Assumptions:
#
# 1) PIT BCVs are defined in the device group
#
# 2) The PIT BCV is established. Note this script will not run
#    if BCV is not established or establishing as this may 
#    may indicate that the SRDF/A outage occurred during a 
#    previously incomplete resunc operation and hence the BCV
#    should NOT be established against a potentially inconsistent
#    R2. 
#
# 
# 1) Verify remote BCV is fully established
# 2) Split remote BCV
# 3) Wait until finished splitting in the backgroup
# 6) Failback RDF
# 9) Wait until all tracks sync'ed.
# 11) Establish remote BCV
# 12) Wait until remote BCV fully established before exiting

##########################################################################
#
# Variables used:
#
my $dgname;		# Device Group Name
#
my $rdf_group;		# Current RDF group being processed
#
my $output;		# Output returned from external commands
#
my $returncode;		# Return code from external commands
#
my $steparg;		# Step # to continue running at.
#
my $statusref;		# Hash reference returned from checkstatuses routine.
#
my $BCVtype;            # Type of BCVs - (BCV or CLONE)
#
my %stepreq;		# Step requirements - what SRDF/BCV should be for this step.
my @stepreq;            # Step requirements - what SRDF/BCV should be for this step.
my $currentstep;        # Which current step are we on.
#
#                       # Where the logging will be saved.
my $LOGdir="/gsmsan04/cob_scripts/output_logs";
#
my $login = getlogin || getpwuid($<) || "Kilroy"; # Who's running the script.
#
my $hostname = (uname())[1]; # Which workstation are we running this on.
#
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
my $datestamp=sprintf("%04d%02d%02d-%02d:%02d:%02d",$year+1900,$mon+1,$mday,$hour,$min,$sec);
#
##########################################################################
#
# Definitions
my %SE_RC_Values=(
    0=>"CLI_C_SUCCESS",
    1=>"CLI_C_FAIL",
    2=>"CLI_C_DB_FILE_IS_LOCKED",
    3=>"CLI_C_SYM_IS_LOCKED",
    4=>"CLI_C_NOT_ALL_SYNCHRONIZED",
    5=>"CLI_C_NONE_SYNCHRONIZED",
    6=>"CLI_C_NOT_ALL_UPDATED",
    7=>"CLI_C_NONE_UPDATED",
    8=>"CLI_C_NOT_ALL_PINGED",
    9=>"CLI_C_NONE_PINGED",
    10=>"CLI_C_NOT_ALL_SYNCHED",
    11=>"CLI_C_NONE_SYNCHED",
    12=>"CLI_C_NOT_ALL_RESTORED",
    13=>"CLI_C_NONE_RESTORED",
    14=>"CLI_C_NOT_ALL_VALID",
    15=>"CLI_C_NONE_VALID",
    18=>"CLI_C_ALREADY_IN_STATE",
    19=>"CLI_C_GK_IS_LOCKED",
    20=>"CLI_C_WP_TRACKS_IN_CACHE",
    21=>"CLI_C_NEED_MERGE_TO_RESUME",
    22=>"CLI_C_NEED_FORCE_TO_PROCEED",
    23=>"CLI_C_NEED_SYMFORCE_TO_PROCEED",
    25=>"CLI_C_NOT_ALL_SPLIT",
    26=>"CLI_C_NONE_SPLIT",
    27=>"CLI_C_NOT_ALL_SYNCINPROG",
    28=>"CLI_C_NONE_SYNCINPROG",
    29=>"CLI_C_NOT_ALL_RESTINPROG",
    30=>"CLI_C_NONE_RESTINPROG",
    31=>"CLI_C_NOT_ALL_SUSPENDED",
    32=>"CLI_C_NONE_SUSPENDED",
    33=>"CLI_C_NOT_ALL_FAILED_OVER",
    34=>"CLI_C_NONE_FAILED_OVER",
    35=>"CLI_C_NOT_ALL_UPDATEINPROG",
    36=>"CLI_C_NONE_UPDATEINPROG",
    37=>"CLI_C_NOT_ALL_PARTITIONED",
    38=>"CLI_C_NONE_PARTITIONED",
    39=>"CLI_C_NOT_ALL_ENABLED",
    40=>"CLI_C_NONE_ENABLED",
    41=>"CLI_C_NOT_ALL_SYNCHRONIZED_AND_ENABLED",
    42=>"CLI_C_NONE_ALL_SYNCHRONIZED_AND_ENABLED",
    43=>"CLI_C_NOT_ALL_SUSP_AND_ENABLED",
    44=>"CLI_C_NONE_SUSP_AND_ENABLED",
    45=>"CLI_C_NOT_ALL_SUSP_AND_OFFLINE",
    46=>"CLI_C_NONE_SUSP_AND_OFFLINE",
    47=>"CLI_C_WONT_REVERSE_SPLIT",
    70=>"CLI_C_NOT_ALL_CONSISTENT",
    71=>"CLI_C_NONE_CONSISTENT",
    77=>"CLI_C_CONSISTENCY_TIMEOUT",
    146=>"CLI_C_NOT_ALL_CONSISTENT_NOINVALIDS",
    147=>"CLI_C_NONE_CONSISTENT_NOINVALIDS",
    148=>"CLI_C_NOT_ALL_SYNCHRONOUS",
    149=>"CLI_C_NONE_SYNCHRONOUS",
    150=>"CLI_C_NOT_ALL_SEMISYNCHRONOUS",
    151=>"CLI_C_NONE_SEMISYNCHRONOUS",
    152=>"CLI_C_NOT_ALL_ASYNCHRONOUS",
    153=>"CLI_C_NONE_ASYNCHRONOUS",
    152=>"CLI_C_NOT_ALL_ACP_WP",
    153=>"CLI_C_NONE_ACP_WP",
    152=>"CLI_C_NOT_ALL_ACP_DISK",
    153=>"CLI_C_NONE_ACP_DISK"
    );
my %SE_RC_Codes=reverse %SE_RC_Values;

#
##########################################################################
#
# Parse input parameters. Exactly 2 arguments expected i.e. $#ARGV = 1
#

# If passed 3 arguments, then looking for special handling.
# 3rd argument should be which step to jump to. No 3rd argument means 
# run like normal, 3rd argument=99 means pick next step automatically.
if ( $#ARGV == 2 ) {
   $steparg = $ARGV[2];
} elsif ( $#ARGV != 1 ) {
   print "Incorrect number of arguments for script failback_async\n";
   print "USAGE: failback_async <DGName> <RDF group>\n";
   exit(1);
}
$dgname = $ARGV[0];
$rdf_group = $ARGV[1];

# Code Started to capture ticket number

sub GetTicket
{
	my $TKTCHECK="FALSE";
        until ($TKTCHECK eq "TRUE")
        {
          print "Please enter ticket number. Press RETURN if none \n";
          my $ticket=<STDIN>;
          chomp($ticket);
          my $size=length($ticket);
        if ($ticket eq "")
                {
                $ticket="none";
                $TKTCHECK="TRUE";
                return $ticket;
                }
        elsif ($ticket =~ m/^(chg|inc|mim)\d{10}$/i)
                {
                print "Valid change/inc number \n";
                $TKTCHECK="TRUE";
                return $ticket;
                }
        elsif ($ticket =~ m/^ctsk\d{10}$/i)
                {
                print "Valid ctsk number \n";
                $TKTCHECK="TRUE";
                return $ticket;
                }
        else
                {
                        print "Invalid Ticket, please enter a valid ticket \n";
                }
        }
}

my $ticket=GetTicket();
print "Ticket number is $ticket \n";


# code ends to capture ticket

my $LOGfile="${LOGdir}/dg_state_${hostname}_${dgname}_${login}_${ticket}_${datestamp}.out";
open (OUT,">>".$LOGfile)
    or die "Cannot open Log File $LOGfile for appending.\n";

# Add Ticket number to logfile
logtofile ("Ticket number for the failback operation is $ticket \n");

if ($#ARGV == 2) {
    logtofile ("Running $0\n version: ${version} \n with DG $ARGV[0], RDF group $ARGV[1], and step $ARGV[2]\n");
} else {
    logtofile("Running $0\n version: ${version} \n with DG $ARGV[0], RDF group $ARGV[1]\n");
}

print "Version: ${version}\n";
print "Logging status to ${LOGfile}\n";

init_stepreq();


##########################################################################
#
# 1) Check remote BCV is synced up first
#
if (checksteprequirements(\%{$stepreq[1]},$steparg,1) == 0)
{
    undef $steparg;
    
    output("\n*** Starting SRDF/A Failback exercise\n");
    output("*** Step 1: Verifying remote BCVs already synced up\n");

    $statusref=checkstatuses("BCV");
    # CHECKING IF TOTAL=SYNCHRONIZED ISN'T 100% ACCURATE. HAVE SEEN WHERE THEY'RE EQUAL BUT TOTAL WAN'T
    # EQUAL TO THE TOTAL NUMBER. (Due to associations not complete.) NEED TO COMPARE TO 
    # TOTAL IN DG AND SHOULD CHECK RETURN CODE OF VERIFY COMMAND.
    if ( ($statusref->{"BCV"}{"Synchronized"} != $statusref->{"BCV"}{"Total"}) ||
	 ($statusref->{"BCV"}{"returncode"} != $SE_RC_Codes{"CLI_C_SUCCESS"} ) )
    {
	output("PIT BCV(s) not already fully synced up. Script exiting.\n");
	exit (2)
    }
    ######################
    # REMOVE AFTER TESTING
    #
    # Specify Step 2 to get to next step
    #
    ######################
    exit;
    output("PROGRAM STILL BEING TESTED! MANUALLY SPECIFY STEP 2 IF READY FOR NEXT STEP\n");
} else {
    output("Skipping step 1\n");
}


##########################################################################
#
# 2) Split remote BCV
#
if (checksteprequirements(\%{$stepreq[2]},$steparg,2) == 0)
{
    undef $steparg;
    
    output("*** Step 2: Splitting remote BCVs\n");
    #symclicommand("${PBRUN} /usr/symcli/bin/symmir -g $dgname -consistent -rdf split -noprompt 2>&1",
    symclicommand("${PBRUN} symmir -g $dgname -consistent -rdf split -noprompt 2>&1",
		  "PIT BCV(s) failed to split. Script exiting.\n", 3);
    ######################
    # REMOVE AFTER TESTING
    #
    # Specify Step 3 to get to next step
    #
    ######################
    output("PROGRAM STILL BEING TESTED! MANUALLY SPECIFY STEP 3 IF READY FOR NEXT STEP\n");
    exit;
} else {
    output("Skipping step 2\n");
}

##########################################################################
#
# 3) Wait until finished splitting in the background
#
if (checksteprequirements(\%{$stepreq[3]},$steparg,3) == 0)
{
    undef $steparg;
    
    if ($BCVtype eq "BCV")
    {
	output("*** Step 3: Waiting for background BCV split for $dgname to finish\n");
	output(" BCVs   BG BCVs   Total\n");
	output("Split  Splitting   BCVs\n");
	$statusref=checkstatuses("BCV");
	while ( ($statusref->{"BCV"}{"Split"} != $statusref->{"BCV"}{"Total"}) ||
		($statusref->{"BCV"}{"SplitBG"} != 0) )
	{
	    my $stat=sprintf("%5d  %8d    %5d                    \r",
			     $statusref->{"BCV"}{"Split"},
			     $statusref->{"BCV"}{"SplitBG"},
			     $statusref->{"BCV"}{"Total"} );
	    output($stat);
	    sleep 5;
	    $statusref=checkstatuses("BCV");
	}
	my $stat=sprintf("%5d  %8d    %5d                    \n",
			 $statusref->{"BCV"}{"Split"},
			 $statusref->{"BCV"}{"SplitBG"},
			 $statusref->{"BCV"}{"Total"} );
	output($stat);
    } else {
	output("*** Step 3: Skipping - Ignoring BCVs splitting as clones in background.\n");
    }
    ######################
    # REMOVE AFTER TESTING
    #
    # Specify Step 6 to get to next step
    #
    ######################
    output("PROGRAM STILL BEING TESTED! MANUALLY SPECIFY STEP 6 IF READY FOR NEXT STEP\n");
    exit;
} else {
    output("Skipping step 3\n");
}

##########################################################################
#
# 6) Failback RDF
#
if (checksteprequirements(\%{$stepreq[6]},$steparg,6) == 0)
{
    undef $steparg;

    output("*** Step 6: Failback RDF for $dgname\n");
    #symclicommand("${PBRUN} /usr/symcli/bin/symrdf -g $dgname -rdfg $rdf_group establish -nop 2>&1",
    symclicommand("${PBRUN} symrdf -g $dgname -rdfg $rdf_group failback -nop 2>&1",
		  "Failback operation failed. Script exiting.\n",2);
    ######################
    # REMOVE AFTER TESTING
    #
    # Specify Step 9 to get to next step
    #
    ######################
    output("PROGRAM STILL BEING TESTED! MANUALLY SPECIFY STEP 9 IF READY FOR NEXT STEP\n");
    exit;
} else {
    output("Skipping step 6\n");
}

##########################################################################
#
# 9) Wait for tracks to completely catch up.
#
if (checksteprequirements(\%{$stepreq[9]},$steparg,9) == 0)
{
    undef $steparg;

    my $lasttracks=0;
    my $lasttime;

    output("*** Step 9: Waiting until $dgname RDF is finished syncing\n");
    output("  R2s     R2s         R2s    Invalid    Est. Time\n");
    output(" Sync'd SyncInProg  Consist  Tracks     Remaining\n");
    $statusref=checkstatuses("SRDF");
    while ($statusref->{"SRDF"}{"Consistent"} != $statusref->{"SRDF"}{"StateTotal"})
    {
	my $thistime=time;
	my $thistracks=$statusref->{"SRDF"}{"TotalTracks"};
	if ($thistracks >= $lasttracks)
	{
	    undef $lasttime;
	}
	if ($lasttime)
	{
	    my $lastcycle=$thistime-$lasttime;
	    my $trackslastcycle=$lasttracks-$thistracks;
	    my $trackspersec=$trackslastcycle/$lastcycle;
	    my $secsremaining=$thistracks/$trackspersec;
	    my $stat=sprintf(" %5d   %5d     %5d   %9d    %02d:%02d:%02d          \r",
		   $statusref->{"SRDF"}{"Synchronized"},
		   $statusref->{"SRDF"}{"SyncInProg"},
		   $statusref->{"SRDF"}{"Consistent"},
		   $statusref->{"SRDF"}{"TotalTracks"},
		   (gmtime $secsremaining)[2,1,0]);
	    output($stat,1);
	} else {
	    my $stat=sprintf(" %5d   %5d     %5d   %9d                            \r",
		   $statusref->{"SRDF"}{"Synchronized"},
		   $statusref->{"SRDF"}{"SyncInProg"},
		   $statusref->{"SRDF"}{"Consistent"},
		   $statusref->{"SRDF"}{"TotalTracks"} );
	    output($stat,1);
	}
	$lasttime=$thistime;
	$lasttracks=$thistracks;
	sleep 5;
	$statusref=checkstatuses("SRDF");
    }
    my $stat=sprintf(" %5d   %5d     %5d   %9d                        \n",
	   $statusref->{"SRDF"}{"Synchronized"},
	   $statusref->{"SRDF"}{"SyncInProg"},
	   $statusref->{"SRDF"}{"Consistent"},
	   $statusref->{"SRDF"}{"TotalTracks"} );
    output($stat);
    ######################
    # REMOVE AFTER TESTING
    #
    # Specify Step 11 to get to next step
    #
    ######################
    output("PROGRAM STILL BEING TESTED! MANUALLY SPECIFY STEP 11 IF READY FOR NEXT STEP\n");
    exit;
} else {
    output("Skipping step 9\n");
}


##########################################################################
#
# 11) Establish remote BCV
#
if (checksteprequirements(\%{$stepreq[11]},$steparg,11) == 0)
{
    undef $steparg;
    
    output("*** Step 11: Incrementally establish remote BCV for $dgname\n");
    #symclicommand("${PBRUN} /usr/symcli/bin/symmir -g $dgname establish -rdf -force -nop 2>&1",
    symclicommand("${PBRUN} symmir -g $dgname establish -rdf -force -nop 2>&1",
		  "Incremental etablish operation failed. Script exiting.\n",2);
    ######################
    # REMOVE AFTER TESTING
    #
    # Specify Step 12 to get to next step
    #
    ######################
    output("PROGRAM STILL BEING TESTED! MANUALLY SPECIFY STEP 12 IF READY FOR NEXT STEP\n");
    exit;
} else {
    output("Skipping step 11\n");
}

##########################################################################
#
# 12) Wait until remote BCV fully established before exiting
#
if (checksteprequirements(\%{$stepreq[12]},$steparg,12) == 0)
{
    undef $steparg;

    my $lasttracks=0;
    my $lasttime;

    output("***  Step 12: Wait until remote BCV fully established for $dgname\n");
    output("  BCVs     BCVs     Invalid    Est. Time\n");
    output(" Sync'd SyncInProg   Tracks    Remaining\n");
    $statusref=checkstatuses("BCV");
    while ($statusref->{"BCV"}{"Synchronized"} != $statusref->{"BCV"}{"Total"})
    {
	my $thistime=time;
	my $thistracks=$statusref->{"BCV"}{"TotalTracks"};
	if ($thistracks >= $lasttracks)
	{
	    undef $lasttime;
	}
	if ($lasttime)
	{
	    my $lastcycle=$thistime-$lasttime;
	    my $trackslastcycle=$lasttracks-$thistracks;
	    my $trackspersec=$trackslastcycle/$lastcycle;
	    my $secsremaining=$thistracks/$trackspersec;
	    my $stat=sprintf(" %5d   %5d    %9d    %02d:%02d:%02d          \r",
		   $statusref->{"BCV"}{"Synchronized"},
		   $statusref->{"BCV"}{"SyncInProg"},
		   $statusref->{"BCV"}{"TotalTracks"},
		   (gmtime $secsremaining)[2,1,0]);
	    output($stat,1);
	} else {
	    my $stat=sprintf(" %5d   %5d    %9d                      \r",
		   $statusref->{"BCV"}{"Synchronized"},
		   $statusref->{"BCV"}{"SyncInProg"},
		   $statusref->{"BCV"}{"TotalTracks"});
	    output($stat,1);
	}
	$lasttime=$thistime;
	$lasttracks=$thistracks;
	sleep 5;
	$statusref=checkstatuses("BCV");
    }
    my $stat=sprintf(" %5d   %5d   %9d                      \n",
	   $statusref->{"BCV"}{"Synchronized"},
	   $statusref->{"BCV"}{"SyncInProg"},
	   $statusref->{"BCV"}{"TotalTracks"} );
    output($stat);
} else {
    output("Skipping step 12.\n Check step argument - provided ($steparg) argument invalid.\n");
    exit (1);
}
    

output("\n***  SRDF/A Resynchronization exercise for device group $dgname completed \n");
exit (0);



#
# Check RDF and BCV status
#
sub checkstatuses
{
    # Get statuses of SRDF and/or BCVs as indicated by argument called with.
    # If only BCV status needed - call with "BCV" argument. If only SRDF status
    # needed call with "SRDF". As appropriate, get output from
    # symrdf verify -enabled -summary and/or  symmir verify -summary.
    # Go through and check counts and use that to identify state of RDF/BCV groups.
    # (Better than dozens of symrdf/symmir verify -state commands.)

    # First go through symrdf output, (if required) and itemize states in variables.
    # Then go through variables, look for states invalid for us here, or inconsistencies
    # that don't look right.
    # Then go through symmir output is required.
    # Then go through symmir variables, looking for states and consistencies.
    # Once we know everything looks "OK", then return the states in a hash to the caller.

    # If "BCV" is passed as an argument, just check BCVs. If "SRDF" is passed, just SRDF.
    my ($check) = @_;

    my %returnhash; # The hash we'll be returning a reference to.

    # Work variables needed for this routine.
    my $output; # Output from the called solutions enabler command.
    my $returncode; # The return code from the solutions enabler command.
    my $space; # Holder for null/space building text keys.
    my $type; # "Type" of state/mode - used for keys into hash.
    my $count; # Count of devs.
    my $RDFstate=""; # The state of the RDF group.
    my $RDFmode=""; # The mode of the RDF group.
    my $BCVstate=""; # The state of the BCVs.
    my @f; # Fields split out of text;
    my $tracks; # Number of invalid tracks.
    my $mb; # Invalid MBs.
    
    if ((! defined $check) || ($check ne "BCV"))
    {
    # First go through symrdf output.
	#$output=`${PBRUN} /usr/symcli/bin/symrdf -g $dgname verify -enabled -summary 2>&1`;
	$output=`${PBRUN} symrdf -g $dgname verify -enabled -summary -rdfg $rdf_group 2>&1`;
	$returncode = ($? >> 8);
# symrdf verify should only fail with a "CLI_C_FAIL" but may have encountered issue.
# Should change instead to check for lack of success.
#	if ($returncode == $SE_RC_Codes{"CLI_C_FAIL"} ) {
# symrdf verify -enabled should succeed with "SUCCESS", "NOT_ALL_ENABLED", or "NONE_ENABLED"
# so fail if it's not one of those three.
	if (($returncode != $SE_RC_Codes{"CLI_C_SUCCESS"}) &&
	    ($returncode != $SE_RC_Codes{"CLI_C_NOT_ALL_ENABLED"}) &&
	    ($returncode != $SE_RC_Codes{"CLI_C_NONE_ENABLED"}) )
	{
#	    output("symrdf verify -enabled -summary returned RC 1.\n");
	    output("symrdf verify -enabled -summary returned RC $returncode.\n");
#	    output($output);
#	    logtofile (" Quitting");
#	    exit(2);
	    quit($output,2);
	}
	$returnhash{"SRDF"}{"returncode"}=$returncode;
	my $statedelim=""; # Holder for null/text as delimiter building $RDFstate;
	my $modedelim=""; # Holder for null/text as delimiter building $RDFmode;
	for (split /^/, $output) {
	    if (/^\s*(Consistent|Synchronized|SyncInProg|Suspend|Split|Failed|R1 Upd|TransIdle|Partition|Invalid)/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<$#f;$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$count=$f[$#f];
		$returnhash{"SRDF"}{$type}=$count;
		if ($count)
		{
		    $RDFstate="$RDFstate$statedelim$type";
		    $statedelim="/";
		}
	    }
	    if ((/^\s*Total\s*[0-9]+\s*$/)&&(! defined $returnhash{"SRDF"}{"StateTotal"})) {
		($type,$returnhash{"SRDF"}{"StateTotal"})=split;
	    }
	    if (/^\s*(Synchronous|Semi-Sync|Asynchronous|Adaptive Copy)/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<$#f;$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$count=$f[$#f];
		$returnhash{"SRDF"}{$type}=$count;
		if ($count)
		{
		    $RDFmode="$RDFmode$modedelim$type";
		    $modedelim="/";
		}
	    }
	    if ((/^\s*Total\s*[0-9]+\s*$/)&&(defined $returnhash{"SRDF"}{"StateTotal"})) {
		($type,$returnhash{"SRDF"}{"ModeTotal"})=split;
		
		if ($returnhash{"SRDF"}{"StateTotal"} != $returnhash{"SRDF"}{"ModeTotal"})
		{
		    output("Inconsistent symrdf counts. Totals not equal.\n");
#		    output("$output");
#		    logtofile (" Quitting");
#		    exit(2);
		    quit($output,2);
		}
	    }
	    if (/^\s*Total.*Invalid/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<($#f-1);$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$tracks=$f[$#f-1];
		$returnhash{"SRDF"}{"TotalTracks"}+=$tracks;
		$mb=$f[$#f];
		$returnhash{"SRDF"}{"TotalMB"}+=$mb;
		$returnhash{"SRDF"}{$type}{"Tracks"}=$tracks;
		$returnhash{"SRDF"}{$type}{"MB"}=$mb;
	    }
	}
	
	# If symrdf verify -enabled returns 0, then it is enabled.
	if ($returncode == $SE_RC_Codes{"CLI_C_SUCCESS"} )
	{
	    $returnhash{"SRDF"}{"Enabled"}=1;
	} else {
	    $returnhash{"SRDF"}{"Enabled"}=0;
	}

	# We have the SRDF State, now see what doesn't look right.
	
	# Resume will only deal with Suspends or Splits.
	# But since this is now a failback script, I'm changing
	# Failed Over into Split and Suspended.
	if ($RDFstate =~ 
#	    /Failed Over|R1 Updated|R1UpdInProg|TransIdle|Partitioned|Invalid/)
	    /Split|Suspended|R1 Updated|R1UpdInProg|TransIdle|Partitioned|Invalid/)
	{
	    output("SRDF status not appropriate for failback.\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	
	# Synchronous or Semi-Synchronous.
	if ($RDFmode =~ /Synchronous/)
	{
	    output("**************************************\n");
	    output("** SRDF IN SYNC MODE. CORRECT ASAP! **\n");
	    output("**************************************\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	
	# Some Consistent and some Sync(ing) or Suspended or Split. That shouldn't be...
	if (($RDFstate =~ /Consistent/) &&
	    ($RDFstate =~ /Sync|Suspended|Split/)) 
	{
	    output("SRDF State seems inconsistent. (consistent+) Please check.\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	
	# Some Sync(ing) and some Suspended or Split. That shouldn't be either...
	if (($RDFstate =~ /Sync/) &&
	    ($RDFstate =~ /Suspended|Split/))
	{
	    output("SRDF State seems inconsistent. (Sync/Suspend/Split) Please check.\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}

	# Some Suspended and some Split. That shouldn't be either...
	if (($RDFstate =~ /Suspend/) &&
	    ($RDFstate =~ /Split/))
	{
	    output("SRDF State seems inconsistent. (Suspend/Split) Please check.\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	
	# Some Async and some Adaptive Copy. That also shouldn't be...
	if (($RDFmode =~ /Asynchronous/) &&
	    ($RDFmode =~ /Adaptive Copy/))
	{
	    output("SRDF Mode seems inconsistent. (async/acp) Please check.\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	
	# Consistency Enabled should only be when in async mode.
	if (($RDFmode =~ /Enabled/) &&
	    ($RDFmode =~ /Adaptive Copy/))
	{
	    output("SRDF Mode seems incorrect. (Enabled/acp) Please check.\n");
	    output("Status: $RDFstate\n");
	    output("Mode: $RDFmode\n");
#	    output("$output");
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
    } #if check eq BCV
    #Now do the same thing for the BCVs.

    # N.B. - NEED TO TAKE INTO ACCOUNT the counts may not be complete. If BCVs aren't associated
    # then the count and total could be reduced. Should use the total in the DG as the "TOTAL".
    if ((! defined $check) || ($check ne "SRDF"))
    {
	#$output=`${PBRUN} /usr/symcli/bin/symmir -rdf -g $dgname verify -synched -summary 2>&1`;
	$output=`${PBRUN} symmir -rdf -g $dgname verify -synched -summary 2>&1`;
	$returncode = ($? >> 8);
# symmir verify should only fail with a "CLI_C_FAIL" but may have encountered an issue here.
# Should change instead to check for lack of success.
#	if ($returncode == $SE_RC_Codes{"CLI_C_FAIL"} ) {
# symmir verify -synched should success with "SUCCESS", "NOT_ALL_SYNCHRONIZED", "NONE_SYNCHRONIZED",
# "NOT_ALL_SYNCHED", or "NON_SYNCHED" so fail if it's not one of these.
	if ( ($returncode != $SE_RC_Codes{"CLI_C_SUCCESS"} ) &&
	     ($returncode != $SE_RC_Codes{"CLI_C_NOT_ALL_SYNCHRONIZED"} ) &&
	     ($returncode != $SE_RC_Codes{"CLI_C_NONE_SYNCHRONIZED"} ) &&
	     ($returncode != $SE_RC_Codes{"CLI_C_NOT_ALL_SYNCHED"} ) &&
	     ($returncode != $SE_RC_Codes{"CLI_C_NONE_SYNCHED"} ) )
	{
#	    output("symmir verify -synched -summary returned RC 1.\n");
	    output("symmir verify -synched -summary returned RC $returncode.\n");
#	    output($output);
#	    logtofile (" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	$returnhash{"BCV"}{"returncode"}=$returncode;
	my $statedelim=""; # Holder for null/text as delimiter building $BCVstate;
	for (split /^/, $output) {
	    if (/^\s*(Sync|Rest|Split|Invalid|Total[^A-Za-z]+[0-9])/)
	    {
		my $bg1; # Text holders for BG split status row.
		my $bg2;
		($type,$count,$bg1,$bg2)=split;
		$returnhash{"BCV"}{$type}=$count;
		if (($count)&&(!/Total/))
		{
		    $BCVstate="$BCVstate$statedelim$type";
		    $statedelim="/";
		}
		if (/Split.*bg/)
		{
		    ($returnhash{"BCV"}{"SplitBG"})=split /\]/, $bg2;
		    if ($returnhash{"BCV"}{"SplitBG"})
		    {
			$BCVstate="$BCVstate${statedelim}SplitBG";
			$statedelim="/";
		    }
		} # if Split.*bg
	    } # if Sync|Rest|Split...
	    if (/^\s*Total.*Invalid/)
	    {
		$type="";
		$space="";
		(@f)=split;
		for (my $i=0;$i<($#f-1);$i++)
		{
		    $type="$type$space$f[$i]";
		    $space=" ";
		}
		$tracks=$f[$#f-1];
		$returnhash{"BCV"}{"TotalTracks"}+=$tracks;
		$mb=$f[$#f];
		$returnhash{"BCV"}{"TotalMB"}+=$mb;
		$returnhash{"BCV"}{$type}{"Tracks"}=$tracks;
		$returnhash{"BCV"}{$type}{"MB"}=$mb;
	    }
	} # for split...
    } # if check eq SRDF

    if (! defined $check) 
    {
	if ($returnhash{"BCV"}{"Total"} != $returnhash{"SRDF"}{"StateTotal"})
	{
	    output("Total # of BCVs and RDF devices in DG are not equal. (".
		   $returnhash{"BCV"}{"Total"}.
		   "!=".
		   $returnhash{"SRDF"}{"StateTotal"}.
		   ")");
#	    output("Correct DG configuration/BCV associations and re-run.");
#	    logtofile(" Quitting.");
#	    exit(2);
	    quit("Correct DG configuration/BCV associations and re-run.",3);
	}
    }

    if (! defined $BCVtype)
    {
	#$output=`${PBRUN} /usr/symcli/bin/symdg show $dgname 2>&1`;
	$output=`${PBRUN} symdg show $dgname 2>&1`;
	$returncode = ($? >> 8);
	if ($returncode != $SE_RC_Codes{"CLI_C_SUCCESS"} )
	{
	    output("symdg show returned RC $returncode");
#	    output($output);
#	    logtofile(" Quitting.");
#	    exit(2);
	    quit($output,2);
	}
	my $rightrdfg=0;
	my $b="";
	for (split /^/, $output)
	{
	    
	    ($b = $_) =~ s/^.*[0-9A-F]{4}\s+(BCV[^\s]*)\s.*$/$1/
		if ((/[0-9A-F]{4}\s+BCV/) && ($b !~ /BCV[^\+]*$/));
		
	    $rightrdfg=1
		if (/RDF \(RA\) Group Number.*:\s+${rdf_group}\s/);
	} # for
	
	if (! $rightrdfg)
	{
	    output("symdg show does not concur with RDF group number.");
	    logtofile(" Quitting.");
	    exit(2);
	}
	if ($b =~ /BCV\+/)
	{ 
	    $BCVtype = "CLONE";
	} elsif ($b =~ /BCV/) {
	    $BCVtype = "BCV";
	} else {
#	    output("Could not identify BCV type from symdg show.");
#	    logtofile(" Quitting");
#	    exit(2);
	    quit("Could not identify BCV type from symdg show.",2);
	}
    } # ! defined $BCVtype
    
    return (\%returnhash);
    
} # sub checkstatuses

####
# checksteprequirements
#
# Take a hash reference to the step requirements and the step arg as parameters.
#
# Perform SRDF/BCV checks against the requirements. 
# If OK to perform this step return 0.
# If not the right step, return 1. 
# If right step, but the requirements don't match, Exit.
#
# If $steparg is undefined, then return 0 - this means prior checks have been run
# and means we're just continuing on.

sub checksteprequirements
{
    my ($reqref,$steparg, $thisstep) = @_;
    my $ret = 0;

    $currentstep=$thisstep;

    if (! defined $steparg)
    {
	return 0;
    }

    if ($reqref->{"step"} != $steparg)
    {
	return 1;
    }

    output("Checking status for step $steparg...\n");

    my $checkstatusref=checkstatuses($reqref->{"check"});

    if (defined $reqref->{"SRDF"})
    {
	my $i;
	foreach $i (@{$reqref->{"SRDF"}})
	{
	    my $ex = $i;
	    $ex =~ s/\>\>([^\<]+)\<\</\$checkstatusref->{\"SRDF\"}{\"$1\"}/g;
	    if (! eval $ex)
	    {
		$ret |= 2;
		logtofile ("Quiting... SRDF Status not correct for step $steparg. ($i)");
		die "SRDF Status not correct for step $steparg. ($i)\n";
	    }
	}
    }
    if (defined $reqref->{"BCV"})
    {
	my $i;
	foreach $i (@{$reqref->{"BCV"}})
	{
	    my $ex = $i;
	    $ex =~ s/\>\>([^\<]+)\<\</\$checkstatusref->{\"BCV\"}{\"$1\"}/g;
	    if (! eval $ex)
	    {
		if (($i =~ /\>\>SplitBG\<\</) && ($BCVtype eq "CLONE"))
		{
		    output("Ignoring BCVs splitting as clones in background.\n");
		} else {
		    $ret |= 2;
		    logtofile ("Quitting... BCV Status not correct for step $steparg. ($i)");
		    die "BCV Status not correct for step $steparg. ($i)\n";
		}
	    }
	}
    }
    return $ret;
}


####
# logtofile
#
# Take a string and output it into the log file with a current timestamp.
#

sub logtofile
{
    my ($outstring) = @_;

    my $now=localtime(time);
    #my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);

    print OUT "$now - $outstring";
}

####
# output
#
# Take a string and ouptput it to the terminal and log it with a timestamp
#

sub output
{
    my ($outstring,$addnewline) = @_;

    print $outstring;
    $outstring.= $addnewline ? "\n" : "";
    logtofile($outstring);
}

# Initialize the %stepreq hash with all the requirements for each step.
#
# The requirements for each step are put together as a list in an array
# that's part of a hash for the step. 
#
# The "step" key to the hash just indicates which step number this should
# be.
#
# The "BCV" and "SRDF" keys indicate the requirements for the current status
# seen when checking the BCVs or the SRDF status.
# In the arrays for these keys, the status of different requirements is checked
# being listed as conditions that are "eval"ed. The simplest comparison would
# be something like "1 == 1" which would always eval to true. In the
# comparisons that are listed, keys into the hashes produced by the BCV
# or SRDF query are included in >> << delimiters. So, for example,
# to check that Step 2 has all BCVs established, SRDF consistency mode is
# "Enabled", everything is set to Async mode and Nothing is consistent:
#
# @{$stepreq[2]{"BCV"}}=(">>Synchronized<< == >>Total<<");
# Means the number of synchronized BCVs is the same as the Total number of
# BCVs - in other words, they are all synchronized.
#
# @{$stepreq[2]{"SRDF"}}=(">>Enabled<< == 1",
#     ">>Asynchronous<< == >>ModeTotal<<",
#     ">>Consistent<< == 0");
# Means symrdf is enabled,
# The number of asynchronous devices is the same as the number of devices
# and the number of Consistent devices is 0.
#
# These can be mostly idenfied by looking at the output of 
# symmir -rdf verify -synched -summary
# and 
# symrdf verify -enabled -summary
# Some keys are modified due to duplication or implication of the return
# code. These can all be checked in the checkstatuses subroutine above

sub init_stepreq
{
    # Step 1 - Nothing special required - this step checks the setup itself.
    $stepreq[1]{"step"}=1;
    
    # Step 2
    $stepreq[2]{"step"}=2;
    undef $stepreq[2]{"check"}; # Check both BCV and SRDF.
    # Step requires BCV count of Synchronized == Total count of BCVs.
    # and SRDF Consistency Enabled == 1,
    # and SRDF Asynchronous devs == Total devs in this mode,
    # and SRDF Failed Over = Total.
    @{$stepreq[2]{"BCV"}}=(">>Synchronized<< == >>Total<<");
    @{$stepreq[2]{"SRDF"}}=(">>Enabled<< == 1",
			    ">>Asynchronous<< == >>ModeTotal<<",
			    ">>Failed Over<< == >>StateTotal<<");

    # Step 3
    $stepreq[3]{"step"}=3;
    undef $stepreq[3]{"check"}; # Check both BCV and SRDF.
    # Step requires that BCV count Synchronized == 0.
    # and SRDF Consistency Enabled == 1,
    # and SRDF Asynchronous devs == Total devs in this mode,
    # and SRDF Failed Over = Total
    @{$stepreq[3]{"BCV"}}=(">>Synchronized<< == 0",
			   ">>SplitBG<< > 0"); # Added to differentiate with step 4.
    @{$stepreq[3]{"SRDF"}}=(">>Enabled<< == 1",
			    ">>Asynchronous<< == >>ModeTotal<<",
			    ">>Failed Over<< == >>StateTotal<<");

    # Step 6
    $stepreq[6]{"step"}=6;
    undef $stepreq[6]{"check"}; # Check both BCV and SRDF.
    # Step requires BCV count of Split == Total
    # and BCV count of SplitBG == 0
    # and SRDF Consistency Enabled
    # and SRDF count Async == Total
    # and SRDF count Failed Over == Total
    @{$stepreq[6]{"BCV"}}=(">>SplitBG<< == 0",
			   ">>Split<< == >>Total<<");
    @{$stepreq[6]{"SRDF"}}=(">>Enabled<< == 1",
			    ">>Asynchronous<< == >>ModeTotal<<",
			    ">>Failed Over<< == >>StateTotal<<");
    
    # Step 9
    $stepreq[9]{"step"}=9;
    undef $stepreq[9]{"check"}; # Check both BCV and SRDF.
    # Step requires BCV count of Split == Total
    # and BCV count of SplitBG == 0
    # and SRDF Consistency Enabled
    # and SRDF count Async == Total
    # and SRDF count Synchronized + SyncInProg + Consistent == Total.
    @{$stepreq[9]{"BCV"}}=(">>SplitBG<< == 0",
			   ">>Split<< == >>Total<<");
    @{$stepreq[9]{"SRDF"}}=(">>Enabled<< == 1",
			    ">>Asynchronous<< == >>ModeTotal<<",
			    "(>>Synchronized<< + >>SyncInProg<< + >>Consistent<<) == >>StateTotal<<",
			    ">>Consistent<< != >>StateTotal<<"); # Added to differentiate with step 10.

    # Step 11
    $stepreq[11]{"step"}=11;
    # Step requires BCV count of Split == Total
    # and BCV count of SplitBG == 0
    # and SRDF Consistency Enabled == 1
    # and SRDF count Async == Total
    # and SRDF count Consistent == Total.
    @{$stepreq[11]{"BCV"}}=(">>SplitBG<< == 0",
			    ">>Split<< == >>Total<<");
    @{$stepreq[11]{"SRDF"}}=(">>Enabled<< == 1",
			     ">>Asynchronous<< == >>ModeTotal<<",
			     ">>Consistent<< == >>StateTotal<<");

    # Step 12
    $stepreq[12]{"step"}=12;
    # Step requires BCV count of Synchronized + SyncInProg == Total
    # and SRDF Consistency Enabled == 1
    # and SRDF count Async == Total
    # and SRDF count Consistent == Total.
    @{$stepreq[12]{"BCV"}}=("(>>Synchronized<< + >>SyncInProg<<) == >>Total<<");
    @{$stepreq[12]{"SRDF"}}=(">>Enabled<< == 1",
			     ">>Asynchronous<< == >>ModeTotal<<",
			     ">>Consistent<< == >>StateTotal<<");

}

# Report which step we're at based on the current SRDF/BCV configuration.

sub whatcurrentstep
{
    my ($checkstatusref) = @_;

    my $currentstep=0;

    for(my $reqidx=1;$reqidx<=$#stepreq;$reqidx++)
    {
	my $reqref=\%{$stepreq[$reqidx]};
	
	my $allgood=1;
	if (defined $reqref->{"SRDF"})
	{
	    my $i;
	    foreach $i (@{$reqref->{"SRDF"}})
	    {
		my $ex = $i;
		$ex =~ s/\>\>([^\<]+)\<\</\$checkstatusref->{\"SRDF\"}{\"$1\"}/g;
		if (! eval $ex)
		{
		    $allgood=0;
		}
	    }
	}
	if (defined $reqref->{"BCV"})
	{
	    my $i;
	    foreach $i (@{$reqref->{"BCV"}})
	    {
		my $ex = $i;
		$ex =~ s/\>\>([^\<]+)\<\</\$checkstatusref->{\"BCV\"}{\"$1\"}/g;
		if (! eval $ex)
		{
		    $allgood=0;
		}
	    }
	}
	
	if ($allgood)
	{
	    $currentstep=$reqidx;
	}
    } # for reqidx=1...
    
    return $currentstep;
} # sub whatcurrentstep

#
# symclicommand - run a given symcli command, make sure that it runs OK and exit with an error if not.
#
sub symclicommand
{
    my ($cmd, $error, $exitcode) = @_;

    my $output=`$cmd`;
    my $returncode = ($? >> 8);
    output($output);
    
    # if command returned a RC that says the symapi database is locked, then try again in a few seconds.
    # We should be able to interrupt with a CTRL-C, so repeat until DB isn't locked.
    while ($returncode == $SE_RC_Codes{"CLI_C_DB_FILE_IS_LOCKED"} )
    {
	my $r=int (rand 60) + 60;
	output("!!! Sleeping $r seconds to try and allow symapi database lock to release. !!!\n");
	output("(Current step: $currentstep)\n");
	sleep $r;

	my $output=`$cmd`;
	$returncode = ($? >> 8);
	output($output);
    }

    # Return code from first run or second possible try should be 0.
    if ($returncode != $SE_RC_Codes{"CLI_C_SUCCESS"} )
    {
#	output($error);
#	exit($exitcode);
	quit($error,$exitcode);
    }
} # sub symclicommand

#
# quit - Report an error and quit - while advising the user how to continue later on.
#
sub quit
{
    my ($error, $exitcode) = @_;

    output($error);
    output("Program Quitting\n");
    output("If issue is temporary, try re-running command at the current step:\n");
    output("\n$0 $ARGV[0] $ARGV[1] $currentstep\n\n");
    output("Otherwise correct issue with this step and try continuing at next step:\n");
    output("\n$0 $ARGV[0] $ARGV[1] ". ($currentstep+1). "\n");
    exit($exitcode);
}