#!/usr/bin/perl5.8.8 
#
#### Few more changes can be made: ################################################
#
#    NewProcessAggregation can be improved, for loops can be inverted. 
#    Although in current scenario this function doesn't get called anywhere.
#
#    Threads are not used due to fact that they don't leave memory untill process completes.
#
#    sum, min, max functions can be improved if used somewhere, currently they are not in use.

###################################################################################


#use strict;

use Switch;
use File::Copy;
use File::Path;
use File::Basename;
use File::Slurp qw(read_file write_file);
use String::Util 'trim';
use Parallel::ForkManager;
# It is use to get current date and time.
use Time::HiRes qw(gettimeofday);
use POSIX "strftime";
use Time::Local;
use Storable qw(store retrieve lock_store lock_retrieve);

sub updateGlobals;
sub updateGlobalsDumb;

####################### GLOBALS & INITIALIZATION  ############################{#

my $UpdateGlobalsSubRef_ = \&updateGlobals;

use constant BASEDIR                => '/usr/local/megha';
use constant MAX_RES_PROCESSES           => 2;
use constant MAX_METRIC_PROCESSES           => 2;
use constant MAX_CUSTOM_AGGR_PROCESSES           => 4;
use constant RAW_DIR_PATH            => 'db/probe/raw';
use constant ROLLUPMETRIC            => 'RollupTo';

use constant RES_RAIDPOOL            => 'raidPool';
use constant RES_RAIDLDEV            => 'raidLdev';

use constant AGGRESNAME_KEY          => 'aggRes';
use constant CUSTOMAGG_KEY           => 'customAggr';

use constant    LITERAL_SERIALNUM       => 'Serial number : ';
use constant    LITERAL_SAMPLINGRATE    => 'sampling rate : ';

# pack, unpack are made constant for 0 and -1.
use constant PACK_FOR_HOLE    => pack('f1',-1);
use constant PACK_FOR_ZERO    => pack('f1',0);
my $BaseDir_ = &BASEDIR;
my $LOGFILE_HANDLE = undef;
my $USAGE_ = "Usage: perl $0 <Input Zip Path> <Site Name> <SVP IP Address> <Start Time> <Output ZIP Path>";

#Process Manager 
# For resource level parallel spawning.
my $PM = new Parallel::ForkManager(&MAX_RES_PROCESSES);

&initLogs();  

if (@ARGV != 5  ) {
    print "$USAGE_\n\n";
    exit 1;
}

# collecting 1st argument which is the absolute path of zip file name to be processed
my $InputZipFile_ = $ARGV[0];  

my $SiteName_ = $ARGV[1]; 
$SiteName_ = substr($SiteName_, index($SiteName_,'=')+1);

my $SVPIP_ = $ARGV[2]; 
$SVPIP_ = substr($SVPIP_, index($SVPIP_,'=')+1);

my $StartTime_ = $ARGV[3];
$StartTime_ = substr($StartTime_, index($StartTime_,'=')+1);

my $OutZipName_ = $ARGV[4];  

# @TODO: following two properties must come as command line args to script from app.properties.
my $CollectionInterval_ = 60; #$ARGV[5];  

# collecting number of samples coming from java from app.properties
my $NumSamples_ = 15; # $ARGV[6];  

my $PERF_SVPIP_LOCK_FILE        = $BaseDir_ . "/" . &RAW_DIR_PATH . "/perf_" . $SVPIP_ . ".lck";
my $PERFSHARE_SVPIP_CONF_FILE   = $BaseDir_ . "/" . &RAW_DIR_PATH . "/perfShare_" . $SVPIP_ . ".txt";
my $CONF_SVPIP_LOCK_FILE        = $BaseDir_ . "/" . &RAW_DIR_PATH . "/conf_" . $SVPIP_ . ".lck";
my $CONFSHARE_SVPIP_PERL_FILE   = $BaseDir_ . "/" . &RAW_DIR_PATH . "/confShare_" . $SVPIP_ . ".pl";

# Dir for Data shared across metrics when preserved for aggregation 
my $SHARED_DIR_PATH_             = $BaseDir_ . "/" . "temp/svp/data/shared/" .  $SVPIP_ ; 

# CHANGES: Directory created for storing temporary files of hash.
my $STORE_RES_DIR_PATH_             = $BaseDir_ . "/" . "temp/svp/data/tmpAggr/"; 

my $PoolVVOLMapRef_ ;
my $ConcatenatedPGMapRef_ ;

my $SerialNum_ = "";
my $ModelUSPV_ = ""; #set to "true" when storage model is USPV
my @ConfList_  = (); #strings to be written to conf file.

#keeps resource to resource signatures for the purpose of aggregation. for eg: PG 1-7 -> (ldev list) {00:00:01,00:00:23,...& so on}  
my %ResAggrMap_ = (); 

#holds resources Sign (eg: raidMPB#53101-MPB-2MC) for resources whose data has already been written in ConfList_ Array
my %ResConfMap_ = ();

my $PerfSVPDir_ =  "$BaseDir_/temp/svp/data/perf/$SVPIP_";
my $PerfDataDir_ = "$PerfSVPDir_/$StartTime_";
my $ProcessingPG_ = "false";

#}##################### GLOBALS & INITIALIZATION END ###########################

############## END SECTION ###############################{#

END {
    #print STDERR "End. Closing FileHandle: $LOGFILE_HANDLE \n";
    close($LOGFILE_HANDLE);
}
#}############ END SECTION ENDS  #############################

############################### MAIN #########################################{#
logMsg("Zip file name: $InputZipFile_\n");
logMsg("Start Time: $StartTime_\nSite Name:$SiteName_\nSVP IP:$SVPIP_\n");

#@TODO handle failure of read_file.
# reading the hash for perf mappings ###
my $wholeContent = read_file("$BaseDir_/lib/svp/hashPerfMapping.pl");
my $resMap  = eval $wholeContent;

undef $wholeContent;
# reading the hash for conf to perf shared data
getConfShareMapRef();

###################################
# Unzipping export tool output file
###################################
my $zipName = basename($InputZipFile_);
$zipName =~ s/\.[^.]+$//;                    #remove extension

my $perfCSVDir = "$BaseDir_/temp/svp/extract/perf/".$zipName;
mkpath $perfCSVDir; #@TODO handle failure of mkpath

if ( system( "unzip -o $InputZipFile_ -d $perfCSVDir" ) ) {
    logMsg("Error: UnZipping raw perf file failed");
    print STDERR "unzipping failed\n";
    exit 1;
}

my $topLevelFolder = "";
my @fileList = glob("$perfCSVDir/*"); 
my $fileName = $fileList[0];
$topLevelFolder = $fileName;

logMsg("topLevelFolder:  $fileName\n\n");

# traversing tmp dir to find perf folder with data (if file is a dir & contains perf in its name)
if ( $fileName =~ m/Perf.*_R_SVP\/[0-9_]+$/ ) {

    processRaidcomFiles($fileName);
		
    # exiting the program once processing and zipping is completed successfully
    if (zipRaidcomFiles()) {  # method returns 1 on completion
        logMsg("Perf RAIDCOM data processing completed successfully\n");
        exit 0;    
		}
		
    logMsg("Perf RAIDCOM data processing incomplete or failed\n");
    exit 1;
	}
elsif ( $fileName =~ m/Perf.*_R_SVP/ ) {
	# this case has been added to handle the case for data coming from 2.0 version probe
	# as there _R files din't  had date level folder, but directly files were were present.
	
    processRaidcomFiles($perfCSVDir);
    	
    # exiting the program once processing and zipping is completed successfully
    if (zipRaidcomFiles()) {  # method returns 1 on completion
        logMsg("Perf RAIDCOM data processing completed successfully\n");
        exit 0;
    }

    logMsg("Perf RAIDCOM data processing incomplete or failed\n");
    exit 1;
}
elsif ( $fileName =~ m/Perf.*\/[0-9_]+$/ ) {
    unzipAll( $topLevelFolder, glob("$topLevelFolder/*"));
}

logMsg("csv copied\n");

# process unzipped perf files using the resMap 
# create temp dir to be zipped  
foreach my $resName (keys %{$resMap}) {

    # Resource level parallelization
    my $childPID = $PM->start and next;
    my $resPath = "$PerfDataDir_/$resName/";
    mkpath $resPath; #@TODO handle failure

    logMsg("resName: $resName\n");
    $ProcessingPG_ = ($resName eq "raidPG") ? "true" : "false";
    my $resourceRef =  $resMap->{$resName} ;  
    my $metricsArrRef = $resourceRef->{"metrics"};

    processMetricList($resName,  $resourceRef, $topLevelFolder);
    logMsg("processMetricList COMPLETED");

# update globals for aggreagted metrics
    if (-e "$SHARED_DIR_PATH_/globals.pl") {
        my $shareMap = retrieve("$SHARED_DIR_PATH_/globals.pl");
        $SerialNum_ = $shareMap->{'SerialNum'};
        $ModelUSPV_ = $shareMap->{'ModelUSPV'};
        $CollectionInterval_ = $shareMap->{'CollectionInterval'};
        $NumSamples_ = $shareMap->{'NumSamples'};
    }

    foreach my $metricHashRef ( @{$metricsArrRef} ) {

        foreach my $metric (keys %{$metricHashRef} ) {
            my $metricRef = $metricHashRef->{$metric};
            my ($subRef, $subArgs) = @{$metricRef}{"subRef", "args"};
            my %resToValueMap = ();

            if($metric eq &ROLLUPMETRIC) {
                logMsg("ROLLUP res: $resName metric: $metric\n");
                my @confList = ();
                processRollupMetric($PerfDataDir_, $resName, $resourceRef, $metricRef,
                \@confList);
                # appendGlobalConfList(\@confList);
                # my $size2KB = (total_size(\@confList)/1024);
                # logMsg("ARR \@confList: $size2KB");
                next;
            }
            elsif (defined $subRef && defined $subArgs) {
				# for calculating one metric using other metrics eg: readIOPS = randomreadIOPS + seqreadIOPS

                logMsg("SUB REF SECTION res: $resName metric: $metric\n");
                my %metricToResValuesMap;
                my @metricArgs = split(',',$subArgs);

                for my $subMetric (@metricArgs) {
                    logMsg("read THIS MAP: $SHARED_DIR_PATH_/$resName/$subMetric.pl\n");

                    next if (! -e "$SHARED_DIR_PATH_/$resName/$subMetric.pl");
                    my $tempResMap1 = retrieve("$SHARED_DIR_PATH_/$resName/$subMetric.pl");
                    $metricToResValuesMap{$subMetric} = $tempResMap1;
                    
                    logMsg("read THIS MAP completed");
                }
				
				# cerating map with resSigns and blank arr for values
                foreach my $key (keys %{$metricToResValuesMap{ $metricArgs[0] }}) {
                    $resToValueMap{$key} = {
                        "value"=> []};
                }
				
				#get all index vals one by one for each metric pass in method and push in this resToValueMap
                my %globalArr ;
                for my $nkey (keys %resToValueMap) {
                    $globalArr{$nkey}={};
                    my %arr;
                    for my $met (@metricArgs) {
                        $arr{$met}={ "value" => [] };
                        @{$arr{$met}->{"value"}} = unpack('f*',${$metricToResValuesMap{$met}}{$nkey}->{"value"});
                    }
                    $globalArr{$nkey}=\%arr;
                }
                
				for my $count (0..$NumSamples_-1) {
                    for my $nkey (keys %resToValueMap) {
                        my @arr;
                        for my $met (@metricArgs) {
                                my $value=$globalArr{$nkey}->{$met}->{"value"}[$count];
                                push(@arr,($value < 0) ? '' : $value );
                        }
                        my $val = &{$subRef}(@arr);
                        push( @{$resToValueMap{$nkey}->{"value"}} , $val);
                    }
                }
					
                undef %{$metricToResValuesMap};
            }
            else {
                next;
            }
					
            my $metricFileName = $resPath."/$metric.txt";
            if( dumpCustomMetricPerf($resName, $metricFileName, \%resToValueMap) ) {
                logMsg("Failed to dump perf file for metric: $metric");
            }
        }
    }
    $PM->finish ;
}
$PM->wait_all_children;

# writing conf data to be shared with Conf processing
writeConfFile($PERF_SVPIP_LOCK_FILE, $PERFSHARE_SVPIP_CONF_FILE);

logMsg("processing completed successfully\n");
##################################################
# Zipping the output folder to final file name
##################################################
chdir($PerfSVPDir_);

if ( system("zip -9 -r $OutZipName_ $StartTime_") ) {
    logMsg("Error: Zipping processed perf file failed");
    print "zipping failed\n";
    exit 1;
}

logMsg("Zipping completed successfully. Output location: $OutZipName_\n");
# cleaning up
rmtree $perfCSVDir;
rmtree $PerfDataDir_;
rmtree $SHARED_DIR_PATH_;
rmtree $STORE_RES_DIR_PATH_;
exit 0;
#}############################# MAIN ENDS ######################################

########################### SUB DEFINITIONS ##########################{#
sub processMetric
{
    my ($resName, $resHashRef, $metric, $metricHashRef, $topLevelFolder) = @_;

    my $aggResName = $resHashRef->{"aggRes"};
    my $metricRef = $metricHashRef->{$metric};
    my $resToValueMap;

    logMsg("in processMetric method $resName \n");
    if (length $metricRef->{"readFrom"}) {
        my ($exOverrideFolder, $exFolderName) = ($metricRef->{"folder"}, $resHashRef->{"folder"});
        my $csvFilesDir = $topLevelFolder . "/";
        $csvFilesDir .= (length $exOverrideFolder) ? $exOverrideFolder : $exFolderName;

        logMsg("$metric \n");

        if(!-d $csvFilesDir) { #dir does'nt exist for husVm,vsp,phyMainPk,phyEsw,uspv-PhyCsw.
            logMsg("csvFilesDir: $csvFilesDir does not exist");
            return 0;
        }

        my @csvFileList = map { basename $_ } glob("'$csvFilesDir/*.csv' '$csvFilesDir/*.CSV'");
        my $exFileName = $metricRef->{"readFrom"};
        my @matchedCsvFileList = grep(/^$exFileName/, @csvFileList);
        return 1 if( scalar @matchedCsvFileList == 0 );

        logMsg("files matched\n");

        my ($signFilter, $signArgs) = ($resHashRef->{"signature"}, $resHashRef->{"signatureArgs"});
        my ($aggSignatureFilter, $aggSignatureArgs) = ($resHashRef->{"aggSignature"},
        $resHashRef->{"aggSignatureArgs"});

        $resToValueMap = processCsv($csvFilesDir, $signFilter, $signArgs, $aggResName,
							$aggSignatureFilter, $aggSignatureArgs, \@matchedCsvFileList);
					
        # my $sizeKB = (total_size($resToValueMap)/1024);
        # logMsg("Map returned by processCsv size: $sizeKB");
        
        # put global share data in a file for aggregated metrics to read
        if (! -e "$SHARED_DIR_PATH_/globals.pl") {
            my $shareMap = {
                'SerialNum' => $SerialNum_,
                'ModelUSPV' => $ModelUSPV_,
                'CollectionInterval' => $CollectionInterval_,
                'NumSamples' => $NumSamples_
            };

            mkpath "$SHARED_DIR_PATH_/" if (! -e "$SHARED_DIR_PATH_/");
            mkpath "$STORE_RES_DIR_PATH_" if (! -e "$STORE_RES_DIR_PATH_/");
            lock_store $shareMap, "$SHARED_DIR_PATH_/globals.pl";
        }

		# save metrics in this map which have preserveForAggr = true
        if($metricRef->{"preserveForAggr"} eq "true") {
            logMsg("saving data for metric: $metric  STARTED");
            mkpath "$SHARED_DIR_PATH_/$resName/";
            store $resToValueMap, "$SHARED_DIR_PATH_/$resName/$metric.pl";
            logMsg("saving data for metric: $metric COMPLETED");
        }

        if ($metricRef->{"customAggr"} eq "true") {
            logMsg("CUSTOM AGGR STARTS");
            my $tempMetricToResValuesMap = { $metric => $resToValueMap};
            my $tempMap = {"metric" => $metric};

            processCustomAggregation($metric, $tempMap, &RES_RAIDPOOL,
            $PerfDataDir_, $tempMetricToResValuesMap);
            logMsg("CUSTOM AGGR ENDS");
        }
    }
					
    if(! %$resToValueMap ) {
        logMsg("Empty resToValueMap!!!");
        return 0;
    }

    my $metricFileName = "$PerfDataDir_/$resName/$metric.txt";
    if( dumpMetricPerf($resName, $metricFileName, $resToValueMap) ) {
        logMsg("Failed to dump perf file for metric: $metric");
    }

    if (-e "$SHARED_DIR_PATH_/resAggrMap.pl" ) {
        my $resAggrMapRef = lock_retrieve("$SHARED_DIR_PATH_/resAggrMap.pl");
        if (! exists $resAggrMapRef->{$aggResName} ) {
            $resAggrMapRef->{$aggResName} = $ResAggrMap_{$aggResName};
            lock_store $resAggrMapRef, "$SHARED_DIR_PATH_/resAggrMap.pl";
        }
    }
    else {
        lock_store \%ResAggrMap_, "$SHARED_DIR_PATH_/resAggrMap.pl";
    }
        
    return 0 if ($resHashRef->{"writeToConf"} ne 'true');
    #dump raidPG resources to conf only if model is USPV
    return 0 if ($resName eq 'raidPG' && $ModelUSPV_ ne 'true');

    my $confMap;
    if ( -e "$SHARED_DIR_PATH_/confShareMap.pl" ) {
        $confMap = lock_retrieve("$SHARED_DIR_PATH_/confShareMap.pl");
        return 0 if ( exists $confMap->{$resName} );
    }
    
    my @confList = ();
    updateConfList($resName, $aggResName, $resToValueMap, \@confList);

    if(@confList) {
        $confMap->{$resName} = \@confList;
        lock_store $confMap, "$SHARED_DIR_PATH_/confShareMap.pl";
    }
}

sub processMetricList {
    my ($resName, $resHashRef, $topLevelFolder) = @_;
	
	# Metric level parallel processing.
    my $MPM = new Parallel::ForkManager(&MAX_METRIC_PROCESSES);
    foreach my $metricHashRef ( @{$resHashRef->{"metrics"}} ) {
        my $MchildPID = $MPM->start and next;

        foreach my $metric (keys %{$metricHashRef} ) {
            logMsg("res: $resName metric: $metric STARTS\n");
            my $metricRef = $metricHashRef->{$metric};
            my ($subRef, $subArgs) = @{$metricRef}{"subRef", "args"};
            next if($metric eq &ROLLUPMETRIC); #skip roll up metric
            #skip metrics which have custom methods defined.
            next if(defined $subRef && defined $subArgs); 
            if( processMetric($resName, $resHashRef,
            $metric, $metricHashRef, $topLevelFolder) ) {
                logMsg("Failed to process Metric: $metric");
            }
            logMsg("res: $resName metric: $metric ENDS\n\n");
        }

        $MPM->finish ;
    }
    $MPM->wait_all_children;

} #end processMetricList

sub updateConfList
{
    my ($resName, $aggResName, $resValMapRef, $confListRef) = @_;

    foreach my $key (keys %$resValMapRef) {
        # added check for not sharing invalid PGs (X100-1) from perf to conf
        next if ( $resName eq 'raidPG' && $key =~ m/.*[XV].*/); 
		
		# eg: exists $ResConfMap_{"raidMpbClpr#53101-MPB-2MC-00"}
        #@TODO candidates for sync $ResConfMap_
        if ( !exists $ResConfMap_{$resName."#".$key} ) {
            $ResConfMap_{$resName."#".$key} = "";
					}
        else { next; }
        
        push (@$confListRef, "R:".$resName."#".$key );
        push (@$confListRef, "a:name=".$key );

        # custom check for raidMpbClpr resource to Add MPB in relation to this resource
        if ($resName eq 'raidMpbClpr') {
            my $mpbClprResName = $key ;
            # extracting mpb name from Mpb-Clpr name to add it in relation
            my $mpbResName = substr ($mpbClprResName, 0, rindex ($mpbClprResName, '-')); 
            push (@$confListRef, "r:raidStorage#$SerialNum_,raidMPB#$mpbResName,");

            # eg: exists $ResConfMap_{"raidMpbClpr#53101-MPB-2MC-00"}
            #@TODO candidates for sync $ResConfMap_
            next if ( exists $ResConfMap_{"raidMPB#$mpbResName"} );

            # add R: Resource for raidMPB, share from perf to conf like others
            push(@$confListRef, ("R:raidMPB#$mpbResName", 
            "a:name=$mpbResName", "r:raidStorage#$SerialNum_,"));

            # adding a key for this signature with blank value to be able to look up in next metric iterations
            #@TODO candidates for sync $ResConfMap_
            $ResConfMap_{"raidMPB#$mpbResName"} = "";
        }
        elsif ($resName eq 'raidMP') {
            # case for adding raidMPB in relation to resource raidMP 
            my $mpResName = $key;
            # extracting mpb name from MP name to add it in relation; eg: raidMP#53086-MP08-2MC) here "2MC" is MPB name
            # for HUS-VM eg: raidMP#210053-MP20-01   where MPB name is MPU-20
            if (length($SerialNum_) == 5 || ( length($SerialNum_) == 6 && $SerialNum_ =~ m/^3[0-9]+/ )) {
                my $mpbResName = substr ($mpResName, rindex ($mpResName, '-')+1, length($mpResName));
                push (@$confListRef, "r:raidStorage#$SerialNum_,raidMPB#$SerialNum_-MPB-$mpbResName,");
            }
            elsif ( length($SerialNum_) == 6 && $SerialNum_ =~ m/^2[0-9]+/) {
                my $mpbResName = substr ($mpResName, index ($mpResName, '-')+3, 2);
                push (@$confListRef, "r:raidStorage#$SerialNum_,raidMPB#$SerialNum_-MPU-$mpbResName,");
            }
        }
        else {
            push (@$confListRef, "r:raidStorage#$SerialNum_");
        }
    }
}

# Those HASH which are not packed can be printed by this function.
sub dumpCustomMetricPerf
{
    my ($resName, $metricFileName, $resValueMapRef) = @_;
    my $METRICFILE;
    return 1 if( ! open($METRICFILE,">$metricFileName") );   

    dumpMetaData($METRICFILE);
    foreach my $key (keys %$resValueMapRef) {
        print $METRICFILE $resName."#".$key . ",";
        my $valStr = join(',',  map { (length $_) ?  customPrint($_) : '' } @{$resValueMapRef->{$key}->{"value"}});
        print $METRICFILE $valStr . "\n";
    }
    close($METRICFILE);
    return 0;
}

# Manage printing of int/float numbers.
sub customPrint
{
    my $val = shift;
    if($val =~ m/^\d+.\d+$/)
    {
        return sprintf("%.2f",$val);
    }
    else {
        return $val;
    }
}

sub dumpMetricPerf
{
    my ($resName, $metricFileName, $resValueMapRef) = @_;
    my $METRICFILE;
    return 1 if( ! open($METRICFILE,">$metricFileName") );   
    dumpMetaData($METRICFILE);
    foreach my $key (keys %$resValueMapRef) {
        print $METRICFILE $resName."#".$key . ",";
        # customPrint is called.
        my $valStr = join(',', map { ($_ != -1) ? customPrint($_) : '' } unpack("f*",$$resValueMapRef{$key}->{"value"}));
        print $METRICFILE $valStr . "\n";
    }
    close($METRICFILE);
    return 0;
}

sub writeConfFile
{
    logMsg ("in writeConfFile method");
    my($lockFile, $confFile) = @_;

	# @TODO use FLOCK for locking this file 
    until(-e $lockFile) {
        my $PerfLock; my $PerfShareFH;
        open($PerfLock, ">$lockFile"); #@TODO Handle Failure

        my $confShareMapRef;
        if (-e "$SHARED_DIR_PATH_/confShareMap.pl") {
            $confShareMapRef = retrieve("$SHARED_DIR_PATH_/confShareMap.pl");
        }
        
		# removing confFile perfShare_<IP>.txt from previous perf processing
        unlink($confFile) if (-e $confFile);
        
        foreach my $resourceName (keys %{$confShareMapRef}) {
            my $contentArrayRef = $confShareMapRef->{$resourceName};
            open ($PerfShareFH, ">>$confFile"); #@TODO handle Failure
        write_file($PerfShareFH, "\n".join("\n", @$contentArrayRef));
            close ($PerfShareFH);
        }
        logMsg("writing Resources completed");
				
        my $resAggrMapRef;
        if ( -e "$SHARED_DIR_PATH_/resAggrMap.pl" ) {
            $resAggrMapRef = retrieve("$SHARED_DIR_PATH_/resAggrMap.pl");

        # added code to share ExternalPG to Ldev list from perf to conf for aggregation in conf
            open ($PerfShareFH, ">>$confFile"); #@TODO handle Failure
            write_file($PerfShareFH, "\nstarts=ExPG to Ldev\n");
            my $raidPGHashRef = $resAggrMapRef->{'raidPG'};
            foreach my $pgName (keys %{$raidPGHashRef}) {
                if ($pgName =~ m/^[0-9]+[-][E][-0-9]+$/) {
                    my @ldevArray = keys %{ $raidPGHashRef->{$pgName} }; # array of ldevs belonging to this PG
                    open ($PerfShareFH, ">>$confFile"); #@TODO handle Failure
                    write_file($PerfShareFH, "$pgName=".join(",", @ldevArray)."\n");
                    close ($PerfShareFH);
                }
            }
            open ($PerfShareFH, ">>$confFile"); #@TODO handle Failure
            write_file($PerfShareFH, "ends\n");
            close ($PerfShareFH);
        }
        close($PerfLock);
    }
    unlink($lockFile);
}

sub initLogs() 
{
    my $svpLogFolder = $BaseDir_ . "/logs/SVPProbe";
    if(!-d $svpLogFolder) {
        mkpath($svpLogFolder);
    }
    my ($seconds, $microseconds) = gettimeofday;
    my $currentTime = strftime("%Y-%m-%d %H:%M:%S", localtime($seconds));
    my($date, $time) = split(' ', $currentTime, 2);

    my $logFile = $svpLogFolder."/SVPProbe.".$date.".log";
    open($LOGFILE_HANDLE, ">>", $logFile) or die "[FATAL ERROR]: Failed to open the LOG file: [$logFile].";
}

#skips first 6 lines of the file
sub updateGlobalsDumb
{
    my ( $CSV_FH ) = @_;
    my $lineNum = 0;

    while( <$CSV_FH> ) {
        return if( 6 == ++$lineNum );
    }
}

#@TODO this is a candidate for sync.
sub updateGlobals # :locked
{
    my ( $CSV_FH ) = @_;

    my $lineNum = 0;
    while( <$CSV_FH> ) {
        chomp;
        $lineNum++;
        last if( 6 == $lineNum );

        switch( $lineNum ) {
            case 2 {
                my $model;
                ($SerialNum_, $model) = ($_ =~ m/@{[ LITERAL_SERIALNUM ]}(.+).*\((.+)\)$/);
                $ModelUSPV_ = "true" if ($model =~ m/RAID600/);
                $SerialNum_ = "3" . $SerialNum_ if ($model =~ m/VSP G1000/);
            }
            case 5 {
                my $tmpStr = substr($_, length &LITERAL_SAMPLINGRATE);
                $CollectionInterval_ = $tmpStr * 60;
            }
        }#switch
    }
    $UpdateGlobalsSubRef_ = \&updateGlobalsDumb;
}

sub processCsvHeader
{
    my ($CSV_FH, $csvFileName, $mpbNameRef, $portNameRef) = @_;

    if($csvFileName =~ m/^PHY_Short_Cache_Usage_Rate_(.*)\.csv/  or 
        $csvFileName =~  m/^PHY_Short_Write_Pending_Rate_(.*)\.csv/) {
        $$mpbNameRef = $1;
    }
    elsif( $csvFileName =~ m/^PortWWN_(.*)_[a-zA-Z]+\.csv/ ) {
        my ($firstChar, $secondChar) = (substr($1,0,1), substr($1,1,1));
        $$portNameRef = "CL$firstChar-$secondChar";
    }

    &{$UpdateGlobalsSubRef_}($CSV_FH);
} #processCsvHeader ends

#method for passing matched csv files and output a resToValueMap
sub processCsv 
{
    my ($csvfilePath, $signFilt, $signArg, $aggRes, 
    $aggSignFilt, $aggSignArgs, $matchedCsvFileListRef) = @_;

    logMsg("in processCsv method\n");

    my %resToValMap = ();
    my $counter = 0;  # counter till 15 for picking 15 vals of each resource
    foreach my $csvFile (@$matchedCsvFileListRef) {
        my $noOfSampleCounter = 0;
        my $mpbName = '';
        my $portName = '';

        my %columnNumToResSignMap;

        my $isResponseMetric = ($csvFile =~ m/Response/) ? 1 : 0;
        # special handling for ResponseTime/Latency metrics, converting from microsecs to millisecs
        # TODO not all ResponseTime metrics are in microsecs, Check Export Tool docs
        my $divisor = ($isResponseMetric) ? 1000 : 1;

        my $CSVFILE;
        open($CSVFILE,"<$csvfilePath/$csvFile") || next;
        processCsvHeader($CSVFILE, $csvFile, \$mpbName, \$portName);
				
        my $currLine;
        while($currLine = readline($CSVFILE)) {
            chomp($currLine);

            if($currLine =~ m/^"No.".*/) { #Resource Header Line
                $counter++;
                %columnNumToResSignMap = (); # map for Column Number => Resource Signature
                $noOfSampleCounter = 0;

                my $index = 0;
                if($signFilt eq '' && $signArg == 0) {
                    $columnNumToResSignMap{$index} = $SerialNum_;

                    $resToValMap{$SerialNum_} = {
                        "value"=> '',
                    };
                    next;
                }

                my @tempArr = split(',',$currLine);
                splice (@tempArr,0,2);  # removing 1st 2 elements of line i.e. "No." and "time"

                foreach my $resourceName (@tempArr) {
                    $resourceName =~ tr/"//d;    #trimQuotes($resourceName);
                    my $resSign = '-'.$resourceName;

                    my @matches = ();
                    if( @matches = ($resourceName =~ m/$signFilt/) ) { 
                        $resSign = "-". join('-',@matches);
                    }

                    #customization for raidMpbClpr resource to add MPB in resourceSign before clprId
                    $resSign = '-'.$mpbName.$resSign if($mpbName ne '' );

                    #customization for raidPortWWN (file name has mpb name not resources column)
                    $resSign = '-'.$portName.$resSign if ($portName ne '');

                    # adding support for Concatenated PG
                    if ($ProcessingPG_ eq "true") {
                        my $tempPgSign = substr ($resSign, 1, length ($resSign)); # removing leading '-' sign

                        if (exists $ConcatenatedPGMapRef_->{$tempPgSign}) {
                            # map contains each singular PG with its combined name
                            my $combinedPgSignature = $ConcatenatedPGMapRef_->{$tempPgSign};
                            $resSign = '-'.$combinedPgSignature;
                        }
                    }

                    $columnNumToResSignMap{$index} = $SerialNum_.$resSign;
                    my %tmpMap = (
                        "value"=> '',
                    );

                    $resToValMap{$SerialNum_.$resSign} = \%tmpMap;

                    my $aggResSign = "";
                    # for aggregate Resource Signature
                    if(defined $aggRes && defined $aggSignFilt && defined $aggSignArgs) {
                        @matches = ();
                        if( @matches = ($resourceName =~ m/$aggSignFilt/) ) {  
                            $aggResSign = "-" . join('-', @matches);
                            $ResAggrMap_{$aggRes}->{$SerialNum_.$aggResSign}->{$SerialNum_.$resSign} = 1;
                        }
                    }

                    $index++;   # increment index counter to count 
                }
                next;   # continue to next line
            } #Resource Header line Done!!!

            if ($currLine =~ m/^"\d+"/) { #Metric Values line
                $noOfSampleCounter++;
                my @tempValArr = split(',',$currLine);
                splice (@tempValArr,0,2);  #removing 1st 2 elements i.e. no. and time

                my $i = 0;
                foreach my $val (@tempValArr) {
                    my $resNameKey = $columnNumToResSignMap{$i};
					
					# Packing efforts are saved if string is zero or one.
                    my $valStrRef = $resToValMap{$resNameKey}->{"value"};
                    $val = ($val < 0) ? -1 : ( ($divisor == 1) ? $val : $val/$divisor );
                    if($val == -1)
                    {
                        $resToValMap{$resNameKey}->{"value"}=$resToValMap{$resNameKey}->{"value"}.&PACK_FOR_HOLE ;
                    }
                    elsif($val == 0) {
                        $resToValMap{$resNameKey}->{"value"}=$resToValMap{$resNameKey}->{"value"}.&PACK_FOR_ZERO ;
                    }
                    else {
                        $resToValMap{$resNameKey}->{"value"}=$resToValMap{$resNameKey}->{"value"}.pack("f1",$val);
                    }
                    $i++;
                }
                next;
            } #Metric Values line Done!!
        } #while readline ends

        close($CSVFILE);
        $NumSamples_ = $noOfSampleCounter;
    }
    return \%resToValMap;
}

# method unzipAll to unzip the csv files and copy to resource folder
sub unzipAll 
{
    my ($path) = @_; #copying the current path to $path & its FileList in tmpFileList
    my @tmpFileList = glob("$path/*");
    my $counter = 0;
    foreach my $perfFileFolder(@tmpFileList) {

        my $perfFile = $perfFileFolder;
        next if (basename($perfFile) =~ m/^\./);  # to skip hidden files starting with "."
        
		# Logic for filtering out non-zip and non-csv files is slightly changed.
        if( -d $perfFile ) { 
            if ($perfFile =~ m/(LDEV_dat)/ ) {
                logMsg("skipping LDEV_DAT in unzip call");
                next;
            }

            opendir(DIR, $perfFile ); 
            my @perfList = grep(/.(zip|ZIP)$/,readdir(DIR));
            closedir(DIR);

            foreach my $zipFile (@perfList) {
               my $retVal = system( "unzip -o $perfFile/$zipFile -d $perfFile/tmp/" );
               if ($retVal) {
                   logMsg("Error: UnZipping raw perf file in unzipAll function for dir having zip files");
                   print "unzipping failed\n";
                   exit 1;
               }

               opendir(DIR, $perfFile."/tmp/" ); 
               my @perfListFromZip = readdir(DIR);
               closedir(DIR);
               foreach my $metricFolder (@perfListFromZip) {
                   if ($metricFolder =~ m/^\./) { # to skip hidden files starting with "."
                       next;
                   }
                   if( -d $perfFile."/tmp/".$metricFolder) {
                       opendir(DIR, $perfFile."/tmp/".$metricFolder ); 
                       my @csvMetricList = grep(/(csv|CSV)$/,readdir(DIR));
                       closedir(DIR);
                       foreach my $csvFile (@csvMetricList) {
                            my $csvFilePath = $perfFile."/tmp/".$metricFolder."/".$csvFile;
                            copy($csvFilePath, $perfFile);
                       }
                   }
               }

			   # delete tmp
               rmtree $perfFile."/tmp/";
            }
        }
        elsif ( $perfFile =~ m/.*\.(zip|ZIP)$/ ) {
            my $ret = system( "unzip -o $perfFile -d $path" );

            if ($ret) {
                logMsg("Error: UnZipping raw perf file in unzipAll function for zip files");
                print "unzipping failed\n";
                exit 1;
            }
					
            my $rootResFolder = substr ($perfFile, 0 , index ($perfFile, '.')); # removing .zip from file name as folder will be of this name
            my $rootResFolderPath = $path."/".$rootResFolder;
            opendir(DIR, $rootResFolderPath );     # root level zips have folder having CSVs.. so unzipping it at root level, so we hacve a folder having all CSVs.
            my @csvList = grep(/(csv|CSV)$/,readdir(DIR));
            closedir(DIR);
            foreach my $csvFile (@csvList) {
                my $csvFilePath = $rootResFolderPath."/".$csvFile;
                copy($csvFilePath, $rootResFolderPath);
            }
			
			# delete zip files at root folder
            unlink($perfFile);   
        }
    }
}

# subroutine to dump meta data. Input args: FILE_HANDLE
sub dumpMetaData {
    my $fileHandle = shift;
    print $fileHandle "%meta\n";
    print $fileHandle "probe_id:".$SiteName_."_".$SVPIP_."\n";
    print $fileHandle "probe_type:SVP\n";
    print $fileHandle "probe_site:$SiteName_\n";
    print $fileHandle "probe_server:$SVPIP_\n";
    print $fileHandle "probe_version:1.0\n";
    print $fileHandle "collection_interval:$CollectionInterval_\n";
    print $fileHandle "number_of_samples:$NumSamples_\n";
    print $fileHandle "%meta\n\n";
}

# sub to calc and return average of values. Input Arg: array of values
# avg logic is changed a bit, even though it is not in use.
sub avg {
    my @arr = @_;
    my $total = 0;
    my $cnt = @arr;
    foreach my $val (@arr) {
        $total += ( $val == '' or $val <= 0 )? next : $val ;
    }
    return $total/$cnt;
}

# sub to calc min among all values passed. Input arg: array of values
sub min {
    my @arr = @_;
    my $min = $arr[0];
    foreach my $val (@arr) {
        if($val == '') {
            next;
        }
        if($val < 0) {
            $val = 0;
        }
        if( $val < $min) {
            $min = $val;
        }
    }
    return $min;
}

# sub to calc max among all values passed.
sub max {
    my @arr = @_;
    my $max = $arr[0];
    foreach my $val (@arr) {
        if($val == '') {
            next;
        }
        if($val < 0) {
            $val = 0;
        }
        if( $val > $max) {
            $max = $val;
        }
    }
    return $max;
}

# This function is used to log a message in the log file. It logs the message in the following format:
# [YYYY-MM-DD HH:MM:ss,SSS]  [SVP-<IP Address>] [Perf_Collection] <Log message>
#
# @param :
# $_[0] - Log message.
#
# @return :
#   Nothing
sub logMsg
{
    my( $file, $line )= ( caller )[1,2];
      my $msg = $_[0];

      my ($seconds, $microseconds) = gettimeofday;
      my $currentTime = "[" . strftime("%Y-%m-%d %H:%M:%S", localtime($seconds)) . "," . $microseconds . "]";

      # Construct the final log message.
      my $LogMsg = $currentTime."  $file($line): [SVP-".$SVPIP_."] [Perf_Collection] ".$msg."\n";

      print $LOGFILE_HANDLE $LogMsg;
    return 0;
}


# sub to process Pool Tier data coming from RAIDCOM
sub processPoolTierData 
{
    my $poolTierDataFile = shift;
    logMsg("pool tier processing started\n");
    open (TIERFILE, "<$poolTierDataFile");
    my $resPath = "$PerfDataDir_/raidPoolTier/";
    mkpath $resPath;
	
    # get serial number either from file name or some coulmn
    # admin_172.17.45.191_210053_perf_poolTier_20140407_230000.txt
    my $tempFileName = substr($poolTierDataFile, rindex($poolTierDataFile, '/')+1 );
    my ($serialNumber) = $tempFileName =~ m/_(\d+)_perf_poolTier/;
    my $collectionInterval = "";
	
    # loop only to obtain the collection interval of the pool tier file for MetaData
    while(my $currLine = readline(TIERFILE)) {

		# header line columns
		# PID  POLS MODE STS DAT TNO TL_RANGE  TD_RANGE  TU_CAP(MB)  TT_CAP(MB) T(%) P(%) R(%) B(%) MM
        if ( $currLine =~ m/^collection_interval:([0-9]+)/ ) {  
            $collectionInterval = $1;
        }
		
		# break from this loop
        last;
    }

    my @counterList = ('lowerLimitIOPH', 'deltaValueIOPH', 'tierUsedCapacity', 'tierTotalCapacity',
    'newAllocFreeSpacePercent', 'performanceWorkingRatio', 'relocationBufferPercent');

    dumpRFilesMetaData($resPath, $collectionInterval, \@counterList);

    while(my $currLine = readline(TIERFILE)) {

		# header line columns
		# PID  POLS MODE STS DAT TNO TL_RANGE  TD_RANGE  TU_CAP(MB)  TT_CAP(MB) T(%) P(%) R(%) B(%) MM
        if ( $currLine =~ m/^collection_interval:([0-9]+)/ ) {  
            next;
        }
        elsif ( $currLine =~ m/^PID[ ]+POLS[ ]+MODE[ ]+STS[ ]+DAT[ ]+TNO[ ]+TL_RANGE[ ]+TD_RANGE[ ]+.*/ ) {
            next;
        }
        else {
            my @arr = split (' ', $currLine);
            my $poolId = $arr[0];
            my $tierNo = $arr[5];
            my $tierLowerLimitIOPH = $arr[6];
            my $tierDeltaValueIOPH = $arr[7];
            my $tierUsedCapMB = $arr[8];
            my $tierTotalCapMB = $arr[9];
            my $tierNewAllocFreeSpacePercent = $arr[10];
            my $tierPerformanceWorkingRatio = $arr[11];
            my $tierRelocationBufferPercent = $arr[13];
					
            # removing leading zeros from PoolId name
            $poolId = sprintf("%d",$poolId);
						
            my $metricFileName = $resPath."/lowerLimitIOPH.txt";
            open(METRICFILE,">>$metricFileName");   
            printf METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,%d\n",$tierLowerLimitIOPH;
            close(METRICFILE);
						
            my $metricFileName = $resPath."/deltaValueIOPH.txt";
            open(METRICFILE,">>$metricFileName");   
            printf METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,%d\n",$tierDeltaValueIOPH;
            close(METRICFILE);
			
            my $metricFileName = $resPath."/tierUsedCapacity.txt";
            open(METRICFILE,">>$metricFileName");   
            my $tierUsedCapGB = ($tierUsedCapMB / 1024);
            printf METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,%.2f\n",$tierUsedCapGB;
            close(METRICFILE);
						
            my $metricFileName = $resPath."/tierTotalCapacity.txt";
            open(METRICFILE,">>$metricFileName");   
            my $tierTotalCapGB = ($tierTotalCapMB / 1024);
            printf METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,%.2f\n",$tierTotalCapGB;
            close(METRICFILE);
			
            my $metricFileName = $resPath."/newAllocFreeSpacePercent.txt";
            open(METRICFILE,">>$metricFileName");   
            print METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,".$tierNewAllocFreeSpacePercent."\n";
            close(METRICFILE);
						
            my $metricFileName = $resPath."/performanceWorkingRatio.txt";
            open(METRICFILE,">>$metricFileName");   
            print METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,".$tierPerformanceWorkingRatio."\n";
            close(METRICFILE);
			
            my $metricFileName = $resPath."/relocationBufferPercent.txt";
            open(METRICFILE,">>$metricFileName");   
            print METRICFILE "raidPoolTier#$serialNumber-$poolId-$tierNo,".$tierRelocationBufferPercent."\n";
            close(METRICFILE);
        }
    }

    logMsg("pool tier processing completed successfully\n");
    close(TIERFILE);
    return "success";     # return success
}

# sub to process Ldev Tier data coming from RAIDCOM
sub processLdevTierData 
{
    my $ldevTierDataFile = shift;
    logMsg("ldev tier processing started\n");

    open (LDEVTIERFILE, "<$ldevTierDataFile");

    my $resPath = "$PerfDataDir_/raidLdevTier/";
    mkpath $resPath;

    # get serial number either from file name or some coulmn
    # admin_172.17.47.66_310063_perf_ldevTier_20140710_050000.txt
    my $tempFileName = substr($ldevTierDataFile, rindex($ldevTierDataFile, '/')+1 );
    my ($serialNumber) = $tempFileName =~ m/_(\d+)_perf_ldevTier/;

    my $collectionInterval = "";

    # loop only to obtain the collection interval of the pool tier file for MetaData
    while(my $currLine = readline(LDEVTIERFILE)) {
        if ( $currLine =~ m/^collection_interval:([0-9]+)/ ) {  
        $collectionInterval = $1;
        }

        # break from this loop
        last;
    }

    my @counterList = ('tierUsedCapacity');

    dumpRFilesMetaData($resPath, $collectionInterval, \@counterList);

    my $metricFileName = $resPath."/tierUsedCapacity.txt";
    open(METRICFILE,">>$metricFileName");   

    my $ldevId = $tierUsedCapGB = $lineCount = 0;
    my %tierMap = ();

    while(my $currLine = readline(LDEVTIERFILE)) {

        if ( $currLine =~ m/^LDEV[ ]+:[ ]+([0-9]+)/ ) {
            $ldevId = ldevSignToHex($1);
            next;
        }
        elsif ( $currLine =~ m/^TIER#(\d)\(MB\)[ ]+:[ ]+([0-9]+)/ ) { 
            my $tierNo = $1;
            $tierUsedCapGB = ( $2 / 1024);
            $tierMap{$tierNo} = $tierUsedCapGB;

            $lineCount++;
            next;
        }
        elsif ( $currLine =~ m/^[\n ]+$/ ) { 
            if ($lineCount >= 1) {
                foreach my $tierNo (keys %tierMap) {
                    printf METRICFILE "raidLdevTier#$serialNumber-$ldevId-$tierNo,%.2f\n",$tierMap{$tierNo}; 
                }
            }

            # re-initializing all variables
            $lineCount = $tierUsedCapGB = 0;     
            $ldevId = "";
        }
    }

    close(METRICFILE);
    close(LDEVTIERFILE);
    logMsg("ldev tier processing completed successfully\n");
    return "success";     # return success
}

# sub to process CLPR data coming from RAIDCOM
sub processClprData 
{
    my $clprDataFile = shift;
    logMsg("CLPR processing started\n");

    open (CLPRFILE, "<$clprDataFile");

    my $resPath = "$PerfDataDir_/raidClpr/";
    mkpath $resPath;

    # get serial number either from file name or some coulmn
    # admin_172.17.47.66_310063_perf_clpr_20140710_050000.txt
    my $tempFileName = substr($clprDataFile, rindex($clprDataFile, '/')+1 );
    my ($serialNumber) = $tempFileName =~ m/_(\d+)_perf_clpr/;

    my $collectionInterval = "";

    # loop only to obtain the collection interval of the pool tier file for MetaData
    while(my $currLine = readline(CLPRFILE)) {

        # header line columns
        # CLPR CLPR_NAME         TC_CAP(MB)  TU_CAP(MB)  WP_CAP(MB)  SF_CAP(MB) U(%) W(%) S(%)
        if ( $currLine =~ m/^collection_interval:([0-9]+)/ ) {  

            $collectionInterval = $1;
        }

        # break from this loop
        last;
    }

    my @counterList = ('utilization','writePendingRate','sideFileUsageRate');

    dumpRFilesMetaData($resPath, $collectionInterval, \@counterList);

    while(my $currLine = readline(CLPRFILE)) {

        # header line columns
        # Serial# LDEV#  SL  CL VOL_TYPE  VOL_Cap(BLK) PID ATTRIBUTE   VOL_Used(BLK) TR TL T#1(MB) T#2(MB) 
        if ( $currLine =~ m/^CLPR[ ]+CLPR_NAME[ ]+TC_CAP\(MB\)[ ]+TU_CAP\(MB\)[ ]+WP_CAP\(MB\)[ ]+SF_CAP\(MB\)[ ]+U\(%\)[ ]+W\(%\)[ ]+S\(%\)/ ) {  

            next;
        }
        else {
            my @arr = split (' ', $currLine);

            my $clprId = $arr[0];        # id is as 001, 003 ... change it to 1, 3 etc.
            my $utilization = $arr[6];   
            my $writePendingRate = $arr[7];
            my $sideFileUsageRate = $arr[8];

            # removing leading zeros from clpr id
            $clprId = sprintf("%d",$clprId);

            my $metricFileName = $resPath."/utilization.txt";
            open(METRICFILE,">>$metricFileName");   
            printf METRICFILE "raidClpr#$serialNumber-$clprId,$utilization\n"; 
            close(METRICFILE);

            my $metricFileName = $resPath."/writePendingRate.txt";
            open(METRICFILE,">>$metricFileName");   
            printf METRICFILE "raidClpr#$serialNumber-$clprId,$writePendingRate\n"; 
            close(METRICFILE);

            my $metricFileName = $resPath."/sideFileUsageRate.txt";
            open(METRICFILE,">>$metricFileName");   
            printf METRICFILE "raidClpr#$serialNumber-$clprId,$sideFileUsageRate\n"; 
            close(METRICFILE);

        }
    }

    logMsg("clpr processing completed successfully\n");
    close(CLPRFILE);
    return "success";     # return success
}

# sub to process Raidcom files 
sub processRaidcomFiles {

    my $fileName = shift;
    my @raidcomFileList = glob("$fileName/*"); 
    my $ret = '';

    foreach my $rFile (@raidcomFileList) {

        logMsg("Raidcom file: $rFile\n");
        if ($rFile =~ m/perf_poolTier/) {
            $ret = processPoolTierData($rFile); 
        }
        elsif ($rFile =~ m/perf_ldevTier/) {
            $ret = processLdevTierData($rFile); 
        }
        elsif ($rFile =~ m/perf_clpr/) {
            $ret = processClprData($rFile); 
        }
        
        if ($ret ne 'success') {
            logMsg("Processing Failed for raidcom file: $rFile\n");
        }
    }
}

# sub to zip the perf folder to output zip file 
sub zipRaidcomFiles 
{
    chdir ("$BaseDir_/temp/svp/data/perf/$SVPIP_");
    my $retVal = system("zip -9 -r $OutZipName_ $StartTime_");

    if ($retVal) {
        logMsg("Error: Zipping processed perf file failed");
        print "zipping failed\n";
        exit 1;
    }

    logMsg("Zipping completed successfully. Output location: $OutZipName_\n");

    # deleting extracted files.
    rmtree $perfCSVDir;

    # deleting temp folder created to keep processed data
    my $tmp = "$BaseDir_/temp/svp/data/perf/$SVPIP_/$StartTime_";
    rmtree $tmp;

    return 1;
}

# common sub to dump meta data for Raidcom files (_R files)
sub dumpRFilesMetaData 
{
    my ($resPath, $collectionInterval, $counterListRef) = @_;

    $CollectionInterval_ = $collectionInterval;  
    $NumSamples_ = 1;

    foreach my $counter (@$counterListRef) {

        my $metricFileName = $resPath."/$counter.txt";
        my $METRICFILE;
        open($METRICFILE,">>$metricFileName");   
        dumpMetaData($METRICFILE);
        close($METRICFILE);
    }
}

# sub to calculate Weighted Average - required for responseTime aggregation from Ldev to Pool
sub calculateWeightedAverage
{
    my ($arrValsRef, $arrWeightsRef) = @_;

    my $num = 0;
    my $totalWeights = 0;

    my @arrVals = @{ $arrValsRef };
    my @arrWeights = @{ $arrWeightsRef };

    my $size = @arrVals;

	# using @$ to convert array ref to array 
    for my $index (0..$size-1) {
        if($arrVals[$index] > 0)
        {
            $num += ($arrVals[$index] * $arrWeights[$index]);
            $totalWeights += $arrWeights[$index];
        }
        else
        { next; }
    }


    my $weightedAvg = "";

    if ($totalWeights > 0 ) {
        $weightedAvg = $num/$totalWeights;
    }

#	return sprintf("%.2f",$weightedAvg);
	return $weightedAvg;
}


# method to get corresponding map from conf share perl file, either PoolToVVolMapRef or Concatenated PG map
sub getConfShareMapRef 
{
    my $wholeShareMap;

	# till conf_svpIP.lck file is present, dnt process anything
    until (-e $CONF_SVPIP_LOCK_FILE){
        my $ConfLock;
        open($ConfLock, ">$CONF_SVPIP_LOCK_FILE");

        if (-e $CONFSHARE_SVPIP_PERL_FILE) {
            logMsg("reading map from conf sharing file: $CONFSHARE_SVPIP_PERL_FILE");
            my $content = read_file($CONFSHARE_SVPIP_PERL_FILE);
            $wholeShareMap  = eval $content;
            undef $content;
        }
        else {
            logMsg("conf sharing map file  $CONFSHARE_SVPIP_PERL_FILE doesn't exist.");
        }

        close($ConfLock);
        unlink($CONF_SVPIP_LOCK_FILE);
        last;
    }

    # fetching PoolVVOLMapRef_ and ConcatenatedPGMapRef_ from wholeShare
    $PoolVVOLMapRef_ = $wholeShareMap->{"poolToVVOLMapRef"};
    $ConcatenatedPGMapRef_ = $wholeShareMap->{"concatenatedPGMap"};
}

sub NewProcessAggregation 
{
    my ($metricName, $metMapRef, $aggResName, 
        $baseResName, $resValMapRef) = @_;
    %$resValMapRef = ();
    my $metResValMapRef;

    return if(!exists $metMapRef->{"metric"});

    return if (! -e "$SHARED_DIR_PATH_/$baseResName/$metricUsed.pl");

    my $content = read_file("$SHARED_DIR_PATH_/$baseResName/$metricUsed.pl");
    my $resMap  = eval $content;
    undef $content;
    $metResValMapRef->{$metricUsed} = $resMap;
    
	# Chances are there to swap loops for performance improvement. If example found, can be done.
    if (exists $metMapRef->{"transformUsing"} ) {
        my $metricUsed = $metMapRef->{"metric"};
        for my $cnt (0..$NumSamples_-1) {
            # PG -> {"1-7" ->{},  "2-11"->{},...} this 1-7 is $aggRes raidPG is $aggResName
            foreach my $aggRes (keys %{ $ResAggrMap_{$aggResName} }) {
                my @baseSignArr = keys %{ $ResAggrMap_{$aggResName}->{$aggRes} };
                my @tempArray = ();
                my $aggResMap = ($resValMapRef->{$aggRes} = {});

                foreach my $baseSign (@baseSignArr) { # base ref sign could be for eg. ldev 00:01:21 

                    if(exists $metResValMapRef->{$metricUsed}->{$baseSign}) {
						# matching resource signatures from resToRes map and res signatures from baseMetric's resToVal map from metResValMapRef
                        # unpack is done to extract packed binaries.
                        push(@tempArray,  ${unpack('f*',$$metResValMapRef{$metricUsed}->{$baseSign}->{"value"})}[$cnt]);
                    }
                }

                my $calcVal =  $metMapRef->{"transformUsing"}(@tempArray);
                $aggResMap->{"value"}[$cnt] = $calcVal;
            }
        }
    }
    else {
	# one to one rollup aggr ... eg: ldevIOPS to pgIOPS for all ldevs that belong to a certain pg
	# PG -> {"1-7" ->{},  "2-11"->{},...} this 1-7 is $aggRes raidPG is $aggResName
        # Loops are swapped for better performance. Unpack calls reduction made a huge difference. 
        foreach my $aggRes (keys %{ $ResAggrMap_{$aggResName} }) {

            my @baseSignArr = keys %{ $ResAggrMap_{$aggResName}->{$aggRes} }; 
            my $metricUsed = $metMapRef->{"metric"};
            my $aggResMap = ($resValMapRef->{$aggRes} = {});
            foreach my $baseSign (@baseSignArr) {
                my @tmpMetricUsedArr=${unpack('f*',$$metResValMapRef{$metricUsed}->{$baseSign}->{"value"})};
                for my $cnt (0..$NumSamples_-1) {

                    if( exists $metResValMapRef->{$metricUsed}->{$baseSign}) {
						# manage resource signatures from resToRes map and res signatures from baseMetric's resToVal map from metResValMapRef
                        my $val=$tmpMetricUsedArr[$cnt];
                        $val=($val < 0)? '' : $val;
                        if($val >= 0 ){
                            $aggResMap->{"value"}[$cnt] += $val;
                        }
                    }
                }
            }
        }
    }
    undef $resMap;
}

sub processCustomAggregation 
{
    my ($metric, $metMapRef, $resName, $outDir, $metResValMapRef) = @_;
    return if (!exists $metMapRef->{"metric"});

    my $resPathAggr = "$outDir/" . $resName;
    mkpath $resPathAggr; #@TODO handle failure

    my $metricUsed = $metMapRef->{"metric"};
    my $weightedAvgMetric = (exists $metMapRef->{"weightedAvgMetric"}) ? $metMapRef->{"weightedAvgMetric"} : "";
   
	# Custom Aggregation parallelization.
    my $PCAPM = new Parallel::ForkManager(&MAX_CUSTOM_AGGR_PROCESSES);

    # for custom weighted/rollup aggregation undefined $metResValMapRef is passed, 
    # for one to one with value is passed 
    my $tempResMap;
    if ($metResValMapRef == undef) {
        if ( -e "$SHARED_DIR_PATH_/".&RES_RAIDLDEV."/$metricUsed.pl"){	
        	$tempResMap = retrieve("$SHARED_DIR_PATH_/".&RES_RAIDLDEV."/$metricUsed.pl");
        	$metResValMapRef->{$metricUsed} = $tempResMap;
    	}
		else {
			logMsg("File doesn't exist: $SHARED_DIR_PATH_/".&RES_RAIDLDEV."/$metricUsed.pl");
			return;
		}
    }

	if ($weightedAvgMetric ne '' && -e "$SHARED_DIR_PATH_/".&RES_RAIDLDEV."/$weightedAvgMetric.pl") {
        $tempResMap = retrieve("$SHARED_DIR_PATH_/".&RES_RAIDLDEV."/$weightedAvgMetric.pl");
        $metResValMapRef->{$weightedAvgMetric} = $tempResMap;
    }	

	# for custom weighted/rollup aggregation undefined $metResValMapRef is passed, 
	foreach my $aggRes (keys %{ $PoolVVOLMapRef_ }) {
		my $PCAchildPID = $PCAPM->start and next;
		next if($aggRes eq '');
		my %resMap = (); 
		my $baseSignArrRef = $PoolVVOLMapRef_->{$aggRes}->{"vvol"}; #for eg: array of vvols belonging to this Pool
		my $noOfVVols = @$baseSignArrRef;
		#######################################
		my @metVals = (\());
		my @wtVals = (\());
		
		if ($weightedAvgMetric ne '') {
			foreach my $baseSign (@$baseSignArrRef) {
				
				@tmp_metric_used_arr=unpack('f*',$$metResValMapRef{$metricUsed}->{$baseSign}->{"value"});
				@tmp_weighted_metric_arr=unpack('f*',$$metResValMapRef{$weightedAvgMetric}->{$baseSign}->{"value"});
				#if RollUpTo metric name map has weightedAvgMetric then its weighted avg is to be calculated
				# for using the given metric as weight, for aggregation
				for my $cnt (0..$NumSamples_-1) {
					if( exists $metResValMapRef->{$metricUsed}->{$baseSign} && exists $metResValMapRef->{$weightedAvgMetric}->{$baseSign})                      {
						push(@{$metVals[$cnt]},$tmp_metric_used_arr[$cnt]);
						push(@{$wtVals[$cnt]},$tmp_weighted_metric_arr[$cnt]);
					}
				}
			}
			undef @tmp_metric_used_arr;
			undef @tmp_weighted_metric_arr;
			if ( $noOfVVols ) {

				for my $cnt (0..$NumSamples_-1) {
					$resMap{$SerialNum_."-".$aggRes}->{"value"}[$cnt] = calculateWeightedAverage($metVals[$cnt], $wtVals[$cnt]); 
				}
			}
		}
		else {
			foreach my $baseSign (@$baseSignArrRef) {

				@tmp_metric_used_arr=unpack('f*',$$metResValMapRef{$metricUsed}->{$baseSign}->{"value"});
				for my $cnt (0..$NumSamples_-1) {
					if( exists $metResValMapRef->{$metricUsed}->{$baseSign})
					{
						my $val=$tmp_metric_used_arr[$cnt];
						#if($val >= 0 ){
						$val=($val < 0 ) ? '' : $val ; 
						$resMap{$SerialNum_."-".$aggRes}->{"value"}[$cnt]  += $val ; 
						#}
					}
				}
			}
			undef @tmp_metric_used_arr;
		}
		
		store \%resMap, $STORE_RES_DIR_PATH_.$aggRes.".txt" ;
		undef %resMap;
		$PCAPM->finish ;
	}
   
    $PCAPM->wait_all_children;

	#as all metrics data related to one Resource metric (LDEV iops, responeTime, etc.) is collected, dump it in customAggr folder
    my $metFileName = $resPathAggr."/$metric.txt";
    my $resMapGlobal = {};
    foreach my $aggRes (keys %{ $PoolVVOLMapRef_ }) {
        my $resmap={};
        $resMap = retrieve($STORE_RES_DIR_PATH_.$aggRes.".txt");
        $resMapGlobal = { %$resMapGlobal,%$resMap };
    }
    if( dumpCustomMetricPerf($resName, $metFileName, $resMapGlobal) ) {
        logMsg("Failed to dump perf file for custom etric: $metric");
    }
    undef %{$metResValMapRef};
}

sub processRollupMetric
{
    my ($outDir, $resName, $resRef, $metricRef, $confEntriesArrayRef) = @_;
    my $aggResName = $resRef->{&AGGRESNAME_KEY};
    my $writeToConf = $resRef->{"writeToConf"};
    my $aggResDir = "$outDir/$aggResName/";
    mkpath $aggResDir; #@TODO handle failure

    foreach my $metric (keys %{$metricRef}) {

        my $customAggr = $metricRef->{$metric}->{&CUSTOMAGG_KEY};
        my %resToValueMap = ();

        if ($customAggr eq "true") {
            my $metResValMapRef = undef;
            processCustomAggregation($metric, $metricRef->{$metric}, &RES_RAIDPOOL,
            $outDir, $metResValMapRef);
            next;
        }
		
		### NOT IN USE, in current data set. Plz have a look once whenever it comes in picture ##############
        NewProcessAggregation($metric, $metricRef->{$metric}, $aggResName, $resName,
        \%resToValueMap);

        ### write to conf ###
        if ($writeToConf eq 'true') {
            my @confList = ();
            updateConfList($resName, $aggResName, \%resToValueMap, \@confList);
            push(@$confEntriesArrayRef, @confList);
        }

        my $metricFile = $aggResDir."/$metric.txt";
        if( dumpCustomMetricPerf($resName, $metricFile, \%resToValueMap) ) {
        }

        undef %resToValueMap;
    }#for each metric ends
}

# sub to convert Ldev number from decimal to ':' separated Hex
sub ldevSignToHex {

    my $ldev = shift;

    my $ldevSignHex = uc (sprintf("%x",$ldev));
    my @tempLdevArr = (0,0,0,0);
    for my $index (0..length($ldevSignHex)-1) {

        my $num = substr($ldevSignHex,$index,1);
        push (@tempLdevArr, $num);
    }
    my $size = @tempLdevArr;
    my $from = $size - 4 ;
    my $ldevName = "";
    my $cntr = 0;
    for my $in ($from..$size-1) {

        $ldevName .= $tempLdevArr[$in];
        $cntr++;
        if($cntr == 2) {
            $ldevName .= ":";
        }
    }

    $ldevSignHex = "00:$ldevName";

    return $ldevSignHex;
}

#}######################### SUB DEFINITIONS END ############################