# Copyright 2008-2014 Cumulus Systems Incorporated.
# All Rights Reserved.

# This file contains the code to process the raw performance data for the IBM XIV Storage and dump it in final MARS format.
#
# @param : Input Zip File Name  - Name of the Zip file containing raw performance data.
# @param : Site Name            - MARS Site Name.
# @param : XIV IP Address       - XIV Storage IP Address
# @param : Start Time           - Start Time.
# @param : Output Zip File Name - Name of the final MARS format performance zip file.
#
# @return : In case of Success, returns 0. In case of Failure, returns 1.
#
# Sample command: perl ibmxivRawPerfDataProcessor.pl <input zip file name> siteName=<siteName> probeServer=<XIV IP Address> startTime=<start time> <output zip file name>

#!/usr/bin/perl -w

# It is used for strict compilation.
use strict;

# It is used to get base file name from the complete file name.
use File::Basename;

# It is used to create a folder recursively.
use File::Path qw(mkpath);

# It is used for getting the contents of a file from one place to another.
use File::Copy;

# It is used to create or remove directory trees.
use File::Path;

# It is used to change the directory path to target location.
use Cwd;

# It is use to get current date and time.
use Time::HiRes qw(gettimeofday);

# Convert date and time information to string.
use POSIX "strftime";

# It is used to efficiently compute time from local and GMT time.
use Time::Local;

# It is used to indicate the path where common module is placed.
use lib "/usr/local/megha/lib/common/perl";

# It is required for logging information.
use commonLogModule;

# It is used for getting probe performance metadata.
use commonProbePerfUtil;

# It is required for getting runtime location of the xivGetConfData.sh file
use commonPropFileModule;

# ----------------------------------------------------------------------------------------------------------------------------------------------------
# GLOBAL Variables
# ----------------------------------------------------------------------------------------------------------------------------------------------------

# This variable is used to store the return status of function calls.
my $status = 0;

# This global array holds resources performance data: ixHost, ixVolume, ixLocalFcPort, ixHostFcPort.
my @perf = ();

# This variable is used to hold the resource type whose performance data is processed.
my $resourceType = "";

# This variable is used to store site name.
my $siteName = "";

# This variable is used to store the current MARS Probe Version number.
my $PROBE_VERSION = "10";

# This variable is used to store storage IP.
my $storageIp = "";

# This variable is used to store storage name.
my $storageName = "";

# This variable is used to store performance data start time.
my $startTime = "";

# This variable is used to hold the list of dump files.
my @dumpFileList = ();

# This variable is used to hold the location of base folder of MARS.
my $baseFolder = "/usr/local/megha";

# This variable is used to hold the name of the various counter types.
my @counterTypes = ();

# This variable is used to hold the name of the variables used to store the complete performance data for each counter.
my @counterList = ();

# This variable is used to store the complete name of log file.
my $logFile = "";

# This variable is used to store the file handle of log file.
my $LOG_HANDLE;

# This variable is used as the FILE handle for current counter performance data file. This file contains counter specific performance data for 15 minutes.
my $PERF_FILE;

# This variable is used to store the location of folder where input file needs to be unzipped.
my $tmpDataDir = "";

# This variable is used to store the absolute path of zip file name to be processed.
my $perfZipFile = "";

# This variable is used to store zip file path for final data.
my $finalFileName = "";

# This variable is used to store the collection interval (in secs) of performance raw data.
my $collectionInterval = 0;

# This variable is used to store the time interval after which performance data will be collected.
my $exportInterval = 0;

# This variable is used to store number of samples to be sent by Probe Appliance in one cycle.
my $noOfSamples = 0;

# This variable is used to store the location of temp directory which needs to be zipped after data processing completed.
my $dirToBeZipped = "";

# This variable is used to store the name (without the extension ".zip") of the file passed as 1st argument.
my $zipFileName = "";

# This variable stores the probe type.
my $probeType = "IbmXiv";

# This variable stores commonLogModule instance.
my $logObj = "";

# This variable is used to store the read IOps performance data counter values for the 15 minute time interval.
my $readIOpsList = "";

# This variable is used to store the write IOps performance data counter values for the 15 minute time interval.
my $writeIOpsList = "";

# This variable is used to store the total IOps performance data counter values for the 15 minute time interval.
my $totalIOpsList = "";

# This variable is used to store the read latency performance data counter values for the 15 minute time interval.
my $readLatencyList = "";

# This variable is used to store the write latency performance data counter values for the 15 minute time interval.
my $writeLatencyList = "";

# This variable is used to store the total latency performance data counter values for the 15 minute time interval.
my $totalLatencyList = "";

# This variable is used to store the read throughput performance data counter values for the 15 minute time interval.
my $readThroughputList = "";

# This variable is used to store the write throughput performance data counter values for the 15 minute time interval.
my $writeThroughputList = "";

# This variable is used to store the total throughput performance data counter values for the 15 minute time interval.
my $totalThroughputList = "";

# ----------------------------------------------------------------------------------------------------------------------------------------------------
# Main function
# ----------------------------------------------------------------------------------------------------------------------------------------------------

# Initialize the environment.
$status = init();
if (0 != $status)
{
    $logObj->error("Call to init() failed.");
    goto EXIT;
}

# Unzip the input performance file containing the raw data.
$status = unzipInputPerfFile();
if (0 != $status)
{
    $logObj->error("Call to unzipInputPerfFile() failed.");
    goto EXIT;
}

# Read each dump file and transform the data into MARS format.
$status = convertRawDataToMarsFormat();
if (0 != $status)
{
    $logObj->error("Call to convertRawDataToMarsFormat() failed.");
    goto EXIT;
}

# Zip the MARS format performance files into the desired location and deletes the temp directory.
$status = zipPerfData();
if (0 != $status)
{
    $logObj->error("Call to zipPerfData() failed.");
    goto EXIT;
}

EXIT:

exit $status;

# ----------------------------------------------------------------------------------------------------------------------------------------------------
# Sub-routines
# ----------------------------------------------------------------------------------------------------------------------------------------------------

# This function is used to initialize the environment to run the Perl script correctly.
#
# @affected global variables :
#    $perfZipFile
#    $siteName
#    $storageIp
#    $startTime
#    $finalFileName
#    @counterTypes
#    @dirToBeZipped
#    $zipFileName
#
# @return :
#   0 if Success
#   1 if Error
sub init
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # This variable stores Log folder path.
    my $logFolder = $baseFolder."/logs";

    # If log folder does not exist then create it.
    if (! (-e $logFolder))
    {
        mkpath($logFolder);
    }

    # Get commonLogModule instance.
    $logObj = commonLogModule->getInstance($logFolder, $probeType, $storageIp);

    # If number of command line arguments are not correct then quit.
    if (5 != ($#ARGV + 1))
    {
        $logObj->error("Number of arguments to ibmxivRawPerfDataProcessor.pl are not correct. Input arguments count = ".($#ARGV + 1));
        $logObj->error("Usage: perl ibmxivRawPerfDataProcessor.pl <input zip file name> siteName=<siteName> probeServer=<XIV IP Address> startTime=<start time> <output zip file name>\n\n");

        $retVal = 1;
        goto EXIT;
    }

    # Collecting 1st argument which is the absolute path of zip file name to be processed.
    $perfZipFile = $ARGV[0];

    $logObj->info("Absolute path of zip file name to be processed is [$perfZipFile].");

    # Collecting site name from 2nd argument.
    $siteName = $ARGV[1];
    $siteName = substr($siteName, index($siteName,'=') + 1);

    $logObj->info("Site name is [$siteName].");

    # Collecting XIV Storage IP from 3rd argument.
    $storageIp = $ARGV[2];
    $storageIp = substr($storageIp, index($storageIp,'=') + 1);

    # Update the final log message information with populated XIV probe id.
    $logObj->setProbeId($storageIp);

    $logObj->info("XIV Storage IP is [$storageIp].");

    # Collecting Performance data start time from 4th argument.
    $startTime = $ARGV[3];
    $startTime = substr($startTime, index($startTime,'=') + 1);

    $logObj->info("Performance data start time is [$startTime].");

    # Collecting output file name in which the processed data must be kept.
    $finalFileName = $ARGV[4];

    $logObj->info("Output file name in which the processed data must be kept is [$finalFileName].");

    # Get commonPropFileModule instance.
    my $propFileObj = commonPropFileModule->getInstance($baseFolder, $probeType, $storageIp, $logObj);

    # Get the collection interval.
    $status = $propFileObj->getProbeProperty("probe.perf.collection.interval.secs", $collectionInterval);
    if (0 != $status)
    {
        $logObj->error("Call to getProbeProperty() failed.");
        $retVal = 1;

        goto EXIT;
    }

    $logObj->info("Perf collection interval is [$collectionInterval].");

    # Get the export interval which indicate the time interval after which performance data will be collected.
    $status = $propFileObj->getProbeProperty("probe.perf.export.interval.secs", $exportInterval);
    if (0 != $status)
    {
        $logObj->error("Call to getProbeProperty() failed.");
        $retVal = 1;

        goto EXIT;
    }

    $logObj->info("Perf export interval is [$exportInterval].");

    # Get the number of performance counters samples to be collected in one batch.
    $noOfSamples = $exportInterval/$collectionInterval;

    $logObj->info("Number of performance sample is [$noOfSamples].");

    # Performance counters for which the performance data will be collected.
    @counterTypes = ('readIOps', 'writeIOps', 'totalIOps', 'readLatency', 'writeLatency', 'totalLatency', 'readThroughput', 'writeThroughput', 'totalThroughput');

    # Storing references of the variables which are used to store the complete performance data for each counter.
    @counterList  = (\$readIOpsList, \$writeIOpsList, \$totalIOpsList, \$readLatencyList, \$writeLatencyList, \$totalLatencyList,
                     \$readThroughputList, \$writeThroughputList, \$totalThroughputList);

    $dirToBeZipped = $baseFolder."/temp/ibmxiv/data/perf/".$storageIp."/".$startTime;

    # Extract zip file name from complete path without extension.
    $zipFileName = basename($perfZipFile);

EXIT:

    return $retVal;
}

# This function is used to unzip the input performance raw files to the required location.
#
# @affected global variables :
#    $tmpDataDir
#    @dumpFileList
#
# @return :
#   0 if Success
#   1 if Error
sub unzipInputPerfFile
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # This variable is used to store the location of folder containing dump files.
    my $dumpFileFolder = "";

    # Unzip the file into the temp location.
    $tmpDataDir = $baseFolder."/temp/ibmxiv/extract/perf/".$storageIp."/".$zipFileName;

    # Create the temp directory.
    mkpath($tmpDataDir);

    $logObj->info("Creating folder: [$tmpDataDir].");

    # Extracting zip file to same location.
    my $retValUnzip = system("unzip -o $perfZipFile -d $tmpDataDir");

    # Check the status of unzip operation.
    if ($retValUnzip)
    {
        $logObj->error("Un-zipping raw performance file [$perfZipFile] failed.");

        $retVal = 1;
        goto EXIT;
    }

    # Read the content of temp directory.
    opendir(DIR, $tmpDataDir);
    my @tmpDirFilesList = readdir(DIR);
    closedir(DIR);

    # Loop over each directory in temp directory.
    foreach my $tmpFile (@tmpDirFilesList)
    {
        # Skip hidden files starting with "."
        if ($tmpFile =~ m/^\./)
        {
            next;
        }

        # Verify the correct date folder.
        if (-d $tmpDataDir."/".$tmpFile && $tmpFile =~ m/^(\d\d\d\d\d\d\d\d)(\_)(\d\d\d\d\d\d)$/)
        {
            # Set the top level folder path under which dump files will be present.
            $dumpFileFolder = $tmpDataDir."/".$tmpFile;

            last;
        }
    }

    # Fill the list of dump file.
    @dumpFileList = glob($dumpFileFolder.'/*.dump');

EXIT:

    return $retVal;
}

# This function is used read each dump file and extract the data into MARS format.
# It internally calls the readDumpFile() and collectMarsPerfData() function.
#
# @affected global variables :
#    @perf
#
# @return :
#   0 if Success
#   1 if Error
sub convertRawDataToMarsFormat
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # Loop over each dump file.
    foreach my $dumpFile (@dumpFileList)
    {
        # Empty the array content.
        undef @perf;

        # Read the current dump file into in-memory data structure.
        $status = readDumpFile($dumpFile);
        if (0 != $status)
        {
            $logObj->error("Call to readDumpFile() failed.");
            $retVal = 1;

            goto EXIT;
        }

        # Check whether the data is successfully read into in-memory data structure.
        if (@perf)
        {
            # Read the in-memory data structure and generate the data for current resource in MARS format.
            $status = collectMarsPerfData();
            if (0 != $status)
            {
                $logObj->error("Call to collectMarsPerfData() failed.");
                $retVal = 1;

                goto EXIT;
            }
        }
    }

EXIT:

    return $retVal;
}

# This function is used to do the following two function:
# 1. Add the comma for each DATA HOLE in performance counter's data.
# 2. Remove the extra data present in performance counter's data.
#
# @param :
#   $_[0] - The number of commas in the performance counter list.
#   $_[1] - The resource name for which the DATA HOLE is found.
#
# @affected global variables :
#   $readIOpsList
#   $writeIOpsList
#   $totalIOpsList
#   $readLatencyList
#   $writeLatencyList
#   $totalLatencyList
#   $readThroughputList
#   $writeThroughputList
#   $totalThroughputList
#
# @return :
#   Nothing
sub correctPerfCounters
{
    # This variable is used to hold the number of commas in performance counter list.
    my $commaCount = $_[0];

    # This variable is used to hold the resource name for which the DATA HOLE is found.
    my $resName = $_[1];

    # Count less than 15 means we have DATA HOLE at the END of data, so appending the comma for each DATA HOLE at the end.
    if ($commaCount < 15)
    {
        my $dataHoleCount = 15 - $commaCount;

        $logObj->warn("[$dataHoleCount] DATA HOLES found for resource [$resName] of type [$resourceType].");

        for (my $i = 0; $i < $dataHoleCount; $i++)
        {
            # Loop over each performance counter list.
            foreach (@counterList)
            {
                ${$_} = ${$_}.",";
            }
        }
    }

    # Count greater than 15 means we have extra data, So removing the extra data from the list. Its a rare case.
    if ($commaCount > 15)
    {
        my $extraDataCount = $commaCount - 15;

        $logObj->warn("Removing [$extraDataCount] extra data counter from the end of performance data of resource [$resName] of type [$resourceType].");

        # Loop over each performance counter list.
        foreach (@counterList)
        {
            # Split the list using comma as delimiter.
            my @counterWithExtraData = split(',', ${$_});

            # Extract only the data up to first 16 comma.(Excluding 16th comma)
            my @counterWithCorrectData = splice(@counterWithExtraData, 0, 16);

            # Get the data in string format.
            ${$_} = join(',', @counterWithCorrectData);
        }
    }
}

# This function is used to read the performance raw dump files into in-memory data structure.
#
# @param:
#    $_[0]: [In] Dump file name.
#
# @affected global variables :
#    $resourceType
#    $storageName
#    @perf
#
# @return :
#   0 if Success
#   1 if Error
sub readDumpFile
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # Create an array to store read IOps data for a particular resource type.
    my @readIOps = ();

    # Create an array to store write IOps data for a particular resource type.
    my @writeIOps = ();

    # Create an array to store total IOps data for a particular resource type.
    my @totalIOps = ();

    # Create an array to store read latency data for a particular resource type.
    my @readLatency = ();

    # Create an array to store write latency data for a particular resource type.
    my @writeLatency = ();

    # Create an array to store total latency data for a particular resource type.
    my @totalLatency = ();

    # Create an array to store read throughput data for a particular resource type.
    my @readThroughput = ();

    # Create an array to store write throughput data for a particular resource type.
    my @writeThroughput = ();

    # Create an array to store total throughput data for a particular resource type.
    my @totalThroughput = ();

    # This variable is used to indicate whether performance data has started or not.
    my $perfDataStart = 0;

    # This variable is used to hold the name of the resources for which the performance data is collected.
    my $resourceName = "";

    # This variable is used to store the minute part of time for previous performance data.
    my $prevDataMinute = undef;

    # This variable is used to store the minute part of time for current performance data.
    my $currDataMinute = undef;

    # This variable is used to store the minute part of the start time at which the performance data get started.
    my $startTimeInStorageTimeZone = undef;

    # Open the file handle to the dump file.
    open(PERF_DUMP, "<", $_[0]) or die $logObj->error("Not able to open the file: [$_[0]].");

    # Loop over each line of the dump file.
    while (<PERF_DUMP>)
    {
        # Remove the new line character.
        $_ =~ s/\r\n//g;
        $_ =~ s/\n//g;

        # Look for "PROBE_META-<resourceType>,<storageName>" string in the file. This string is the indicator of performance data for particular
        # resource type coming from particular storage.
        if ($_ =~ /^PROBE_META-(\w+),(.+)$/)
        {
            $resourceType = $1;
            $storageName = $2;

            # Remove trailing spaces.
            $storageName =~ s/\s+$//;
        }

        # Look for "START-<resourceName>" string in the file. This string is the start indicator of performance data for particular resource.
        if ($_ =~ /^\*\* START-(.+) \*\*$/)
        {
            $perfDataStart = 1;
            $resourceName = $1;

            # Remove trailing spaces.
            $resourceName =~ s/\s+$//;

            # Replace space by underscore in resource name.
            $resourceName =~ s/ /_/g;

            # Skip this line as it doesn't contain any useful data.
            next;
        }

        if ($_ =~ /^(\s*)(.+)>>STOP$/)
        {
            $perfDataStart = 0;

            # Check for useful data.
            if ($readIOpsList eq '' || $writeIOpsList eq '' || $totalIOpsList eq '' || $readLatencyList eq '' || $writeLatencyList eq '' ||
                $totalLatencyList eq '' || $readThroughputList eq '' || $writeThroughputList eq '' || $totalThroughputList eq '')
            {
                # Skip as it doesn't contain any useful data.
                next;
            }

            # Check for DATA HOLE in performance data. Calculate the number of commas in the counter list (any list).
            my @matches = $readIOpsList =~ /(,)/g;
            my $count = @matches;

            # Absence of exactly 15 commas in the counter list indicate DATA HOLE.
            if ($count != 15)
            {
                # Properly make the DATA HOLE entry in the performance counter list using the following function call.
                correctPerfCounters($count, $resourceName);
            }

            # Create an array to store the resource read IOps data and insert the resource name and read IOps data list.
            my @resReadIOps = ();
            push(@resReadIOps, $resourceName);
            push(@resReadIOps, $readIOpsList);

            # Pass the reference to the array which stores the resource read IOps data into the global read IOps data array.
            push(@readIOps, \@resReadIOps);

            # Create an array to store the resource write IOps data and insert the resource name and write IOps data list.
            my @resWriteIOps = ();
            push(@resWriteIOps, $resourceName);
            push(@resWriteIOps, $writeIOpsList);

            # Pass the reference to the array which stores the resource write IOps data into the global write IOps data array.
            push(@writeIOps, \@resWriteIOps);

            # Create an array to store the resource total IOps data and insert the resource name and total IOps data list.
            my @resTotalIOps = ();
            push(@resTotalIOps, $resourceName);
            push(@resTotalIOps, $totalIOpsList);

            # Pass the reference to the array which stores the resource total IOps data into the global total IOps data array.
            push(@totalIOps, \@resTotalIOps);

            # Create an array to store the resource read latency data and insert the resource name and read latency data list.
            my @resReadLatency = ();
            push(@resReadLatency, $resourceName);
            push(@resReadLatency, $readLatencyList);

            # Pass the reference to the array which stores the resource read latency data into the global read latency data array.
            push(@readLatency, \@resReadLatency);

            # Create an array to store the resource write latency data and insert the resource name and write latency data list.
            my @resWriteLatency = ();
            push(@resWriteLatency, $resourceName);
            push(@resWriteLatency, $writeLatencyList);

            # Pass the reference to the array which stores the resource write latency data into the global write latency data array.
            push(@writeLatency, \@resWriteLatency);

            # Create an array to store the resource total latency data and insert the resource name and total latency data list.
            my @resTotalLatency = ();
            push(@resTotalLatency, $resourceName);
            push(@resTotalLatency, $totalLatencyList);

            # Pass the reference to the array which stores the resource total latency data into the global total latency data array.
            push(@totalLatency, \@resTotalLatency);

            # Create an array to store the resource read throughput data and insert the resource name and read throughput data list.
            my @resReadThroughput = ();
            push(@resReadThroughput, $resourceName);
            push(@resReadThroughput, $readThroughputList);

            # Pass the reference to the array which stores the resource read throughput data into the global read throughput data array.
            push(@readThroughput, \@resReadThroughput);

            # Create an array to store the resource write throughput data and insert the resource name and write throughput data list.
            my @resWriteThroughput = ();
            push(@resWriteThroughput, $resourceName);
            push(@resWriteThroughput, $writeThroughputList);

            # Pass the reference to the array which stores the resource write throughput data into the global write throughput data array.
            push(@writeThroughput, \@resWriteThroughput);

            # Create an array to store the resource total throughput data and insert the resource name and total throughput data list.
            my @resTotalThroughput = ();
            push(@resTotalThroughput, $resourceName);
            push(@resTotalThroughput, $totalThroughputList);

            # Pass the reference to the array which stores the resource total throughput data into the global total throughput data array.
            push(@totalThroughput, \@resTotalThroughput);

            # Loop over each performance counter list.
            foreach (@counterList)
            {
                # Reset the list so that the data can't be repeated.
                ${$_} = "";
            }

            # Reset the current and previous values to be used for next resource.
            $prevDataMinute = undef;
            $currDataMinute = undef;
        }

        # Check whether the actual performance data is started or not.
        if (1 == $perfDataStart)
        {
            # Check for the start time in the command use to get the performance data.
            if (/^statistics_get start=(\d\d\d\d)(\-)(\d\d)(\-)(\d\d)(\.)(\d\d)(\:)(\d\d)(\:)(\d\d)/)
            {
                $startTimeInStorageTimeZone = $9;
            }
            # Check for single minute performance data.
            elsif (/^(\d\d\d\d)(\-)(\d\d)(\-)(\d\d)( )(\d\d+)(\:)(\d\d)(\:)(\d\d)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)(\s+)(\d+)/)
            {
                my $readIOps = $13 + $19 + $25 + $31 + $37 + $43 + $49 + $55;
                my $writeIOps = $61 + $67 + $73 + $79 + $85 + $91 + $97 + $103;

                my $readLatency = 0;
                my $writeLatency = 0;

                if ($readIOps == 0)
                {
                    $readLatency = 0;
                }
                else
                {
                    $readLatency = (($13 * $15) + ($19 * $21) + ($25 * $27) + ($31 * $33) +
                                    ($37 * $39) + ($43 * $45) + ($49 * $51) + ($55 * $57)) / ($readIOps);

                    # As we have to show read latency unit as millisecond in UI so converting the value from microsecond to millisecond.
                    $readLatency = $readLatency / 1000;
                }

                if ($writeIOps == 0)
                {
                    $writeLatency = 0;
                }
                else
                {
                    $writeLatency = (($61 * $63) + ($67 * $69) + ($73 * $75) + ($79 * $81) +
                                     ($85 * $87) + ($91 * $93) + ($97 * $99) + ($103 * $105)) / ($writeIOps);

                    # As we have to show write latency unit as millisecond in UI so converting the value from microsecond to millisecond.
                    $writeLatency = $writeLatency / 1000;
                }

                my $readThroughput = $17 + $23 + $29 + $35 + $41 + $47 + $53 + $59;
                my $writeThroughput = $65 + $71 + $77 + $83 + $89 + $95 + $101 + $107;

                # Calculate the total values of resource criteria from criteria read and write data.
                my $totalIOps = $readIOps + $writeIOps;
                my $totalLatency = $readLatency + $writeLatency;
                my $totalThroughput = $readThroughput + $writeThroughput;

                # Read the minute attribute of the time.
                my $minuteInTime = $9;

                # This variable holds the no. of comma to be inserted between data values in case of data hole.
                my $noOfComma = ",";

                if (defined $prevDataMinute)
                {
                    $prevDataMinute = $currDataMinute;
                }
                # First time
                else
                {
                    # Store the start time use in command to get the performance data into the variable used to calculate the DATA HOLE.
                    # Handling the corner case of HH:00 time.
                    if ($startTimeInStorageTimeZone == 0)
                    {
                        $prevDataMinute = 59;
                    }
                    else
                    {
                        $prevDataMinute = $startTimeInStorageTimeZone - 1;
                    }
                }

                $currDataMinute = $minuteInTime;

                my $diff = $currDataMinute - $prevDataMinute;

                # It means the same time is repeated in data. So ignore it.
                if ($diff == 0)
                {
                    next;
                }

                # Handling the corner case of change in hour boundary.
                if ($diff < 0)
                {
                    $diff = ($currDataMinute + 60) - $prevDataMinute;
                }

                # Diff 1 is normal, it represent consecutive values of data.
                if ($diff != 1)
                {
                    # diff = (no. of comma - 1)
                    for (my $i = 1; $i < $diff; $i++)
                    {
                        $noOfComma = $noOfComma.",";
                    }
                }

                # Store the data into the comma separated list.
                $readIOpsList = $readIOpsList.$noOfComma.$readIOps;
                $writeIOpsList = $writeIOpsList.$noOfComma.$writeIOps;
                $totalIOpsList = $totalIOpsList.$noOfComma.$totalIOps;

                $readLatencyList = $readLatencyList.$noOfComma.$readLatency;
                $writeLatencyList = $writeLatencyList.$noOfComma.$writeLatency;
                $totalLatencyList = $totalLatencyList.$noOfComma.$totalLatency;

                $readThroughputList = $readThroughputList.$noOfComma.$readThroughput;
                $writeThroughputList = $writeThroughputList.$noOfComma.$writeThroughput;
                $totalThroughputList = $totalThroughputList.$noOfComma.$totalThroughput;
            }
            elsif (/^No statistics for the given criteria$/)
            {
                # Ignore the resource.
                # Also write the resource type in log.
                $logObj->warn("No statistics for the given criteria [$resourceName].");
            }
            elsif (/^Error:( +)(\w+)$/)
            {
                # Log the error.
                # Also write the resource type in log.
                $logObj->error("Got the error [$1] while getting performance data for [$resourceName].");
            }
            else
            {
                # Current line is of no use. So ignore it.
            }
        }
    }

    # Create array to store all given resource data.
    my @resResourceType = ();

    # Store the reference in global resource type array.
    push(@resResourceType, \@readIOps);
    push(@resResourceType, \@writeIOps);
    push(@resResourceType, \@totalIOps);

    push(@resResourceType, \@readLatency);
    push(@resResourceType, \@writeLatency);
    push(@resResourceType, \@totalLatency);

    push(@resResourceType, \@readThroughput);
    push(@resResourceType, \@writeThroughput);
    push(@resResourceType, \@totalThroughput);

    # Pass reference to global resource type array into global performance data array.
    push(@perf, \@resResourceType);

    close (PERF_DUMP);

EXIT:

    return $retVal;
}

# This function is used to dump the processed performance data in output file.
#
# @affected global variables :
#    None
#
# @return :
#   0 if Success
#   1 if Error
sub collectMarsPerfData
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # This variable is used to store the complete path of folder where current resource specific data will be stored.
    my $resourceFolder = $dirToBeZipped."/".$resourceType;

    # Create the resource name folder.
    unless (defined eval {mkpath($resourceFolder)})
    {
        $logObj->error("Not able to create folder: [$resourceFolder].");
        $retVal = 1;

        goto EXIT;
    }

    # This variable stores IBM XIV Storage performance metadata.
    my $metaData = "";

    # Get commonPerlUtil instance.
    my $probePerfUtilObj = commonProbePerfUtil->getInstance($baseFolder, $probeType, $storageIp, $logObj);

    # Get the probe metadata in final MARS performance file format.
    $probePerfUtilObj->getFormattedProbePerfMetadata($siteName, $PROBE_VERSION, $collectionInterval, $noOfSamples, $metaData);

    # This temp variable is used increment purpose.
    my $idx = 0;

    # This temp variable is used to hold the counter names.
    my $counterName = "";

    # Loop over each entry in global performance array.
    foreach (@perf)
    {
        # Set the index to zero position.
        $idx = 0;

        # Loop through the list of performance counters for the current Resource.
        foreach (@$_)
        {
            # Extract the counter name from the list containing all the counter type names.
            $counterName = $counterTypes[$idx];

            # Create the performance file for the current counter.
            my $perfFile = $resourceFolder."/".$counterName.".txt";

            # Open the file.
            open($PERF_FILE, ">", $perfFile) or die $logObj->error("Not able to open the file: [$perfFile].");

            # Dump the metadata for the IBM XIV probe in the performance file.
            print $PERF_FILE $metaData;

            # Loop over each resource entry in the given counter.
            foreach (@$_)
            {
                print $PERF_FILE "$resourceType#$storageName^";

                foreach (@$_)
                {
                    print $PERF_FILE "$_";
                }

                print $PERF_FILE "\n";
            }

            # Close file.
            close($PERF_FILE);

            # Increment the index.
            $idx++;
        }
    }

EXIT:

    return $retVal;
}

# This function is used to zip the MARS format performance files into the desired location and deletes the temp directory.
#
# @affected global variables :
#    None
#
# @return :
#   0 if Success
#   1 if Error
sub zipPerfData()
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # Change the current working directory to the target location.
    chdir($baseFolder."/temp/ibmxiv/data/perf/".$storageIp);

    # Zip the files.
    my $zipRetVal = system("zip -9 -r $finalFileName $startTime");

    # Check zipping operation status.
    if ($zipRetVal)
    {
        $logObj->error("Zipping processed performance file present in folder [$baseFolder/temp/ibmxiv/data/perf/$storageIp/$startTime] failed");

        $retVal = 1;
        goto EXIT;
    }

    # Deleting extracted files.
    rmtree($tmpDataDir);

    # Deleting temp folder created to keep processed data.
    rmtree($dirToBeZipped);

EXIT:
    return $retVal;
}