#!/usr/bin/perl -w
# Activate the use warnings pragma with the "-w" flag.

# Copyright 2008-2014 Cumulus Systems Incorporated.
# All Rights Reserved.

# This Script contains code to convert Raw Linux data into MARS format.
# This script called by CRON at a fixed frequency of 1 minute.
# If instance of this script is already running then it will quit.
#
# @param: Property File Path
#
# @return: In case of success, returns 0. In case of failure, returns 1.
#
# Sample command: perl processRawData.pl <Property File Path>

use strict;

# Parse file paths into directory, file name and suffix.
use File::Basename;

# Copy files or file handles.
use File::Copy;

# Create or remove directory trees.
use File::Path;

# Gives dates on the Gregorian Calendar.
use Date::Gregorian;

# Used to dump Perl data structures. Suitable for both printing and eval.
use Data::Dumper;

# Get pathname of current working directory.
use Cwd;

use FileHandle;

# It is used to get the current date and time.
use Time::HiRes qw(gettimeofday);

# The POSIX module permits you to access all (or nearly all) the standard POSIX 1003.1 identifiers.
use POSIX "strftime";

# Host resource name.
my $HOST_PREFIX = "lHost";

# Host CPU resource name.
my $HOST_CPU_PREFIX = "lHostCPU";

# Host Disk resource name.
my $HOST_DISK_PREFIX = "lHostDisk";

# Host Network resource name.
my $HOST_NET_PREFIX = "lHostIfc";

# Temporary configuration file name.
my $TEMP_CONF_FILE_NAME = "hostConfigData.txt";

# Map for keeping configuration attributes.
my %confAttributeMap = ();

# Maps of our metric names.

# Following attributes are related to processors. Collected from mpstat command.
my %MPSTAT_METRIC_MAP = (
    "%usr" => "l_user",
    "%user" => "l_user",
    "%nice" => "l_nice",
    "%sys" => "l_sys",
    "%iowait" => "l_iowait",
    "%irq" => "l_irq",
    "%soft" => "l_soft",
    "%steal" => "l_steal",
    "%idle" => "l_idle",
    "intr/s" =>  "l_intrps"
);

# Following attribute are related to processes, memory, paging, block IO, traps, and cpu activity. Collected from vmstat command.
my %VMSTAT_METRIC_MAP = (
    "r" => "l_r",
    "b" => "l_b",
    "swpd" => "l_swpd",
    "free" => "l_free",
    "inact" => "l_inact",
    "active" => "l_active",
    "si" => "l_si",
    "so" => "l_so",
    "bi" =>  "l_bi",
    "bo" =>  "l_bo",
    "cs" =>  "l_cs"
);

# Following attribute are related to central Processing Unit (CPU) statistics and input/out-put statistics for devices and partitions.
# Collected from iostat command.
my %IOSTAT_METRIC_MAP = (
    "r/s" => "l_rps",
    "w/s" => "l_wps",
    "rMB/s" => "l_rkbps",
    "wMB/s" => "l_wkbps",
    "rkB/s" => "l_rkbps",
    "wkB/s" => "l_wkbps",
    "avgrq-sz" => "l_avgrq-sz",
    "avgqu-sz" => "l_avgqu-sz",
    "await" => "l_await",
    "svctm" => "l_svctm",
    "%util" =>  "l_util",

    # Our own created
    "l_tps" =>  "l_tps",
    "l_mbps" =>  "l_mbps",
    "l_rmbps" =>  "l_rmbps",
    "l_wmbps" =>  "l_wmbps"
);

# Following attribute are network IO.
my %NETSTAT_METRIC_MAP = (
    "Receive bytes" => "l_rxkbps",
    "Receive packets" => "l_rxpckps",
    "Receive errs" => "l_rxerrps",
    "Receive drop" => "l_rxdropps",
    "Receive fifo" => "l_rxfifops",
    "Receive frame" => "l_rxframeps",
    "Receive compressed" => "l_rxcmpps",
    "Receive multicast" => "l_rxmcstps",

    "Transmit bytes" => "l_txkbps",
    "Transmit packets" => "l_txpckps",
    "Transmit errs" => "l_txerrps",
    "Transmit drop" => "l_txdropps",
    "Transmit fifo" => "l_txfifops",
    "Transmit colls" => "l_txcollsps",
    "Transmit compressed" => "l_txcmpps",
    "Transmit carrier" => "l_txcarrps",

    # Our own created
    "l_xmbps" =>  "l_xmbps",
    "l_txmbps" =>  "l_txmbps",
    "l_rxmbps" =>  "l_rxmbps",
    "l_pckps" =>  "l_pckps"
);

# It is used to indicate the path where common module is placed.
use lib "/usr/local/megha/lib/common/perl";

# It is required for logging information.
use commonLogModule;

# This variable stores commonLogModule instance.
my $logObj = "";

# This variable stores the probe type.
my $probeType = "Linux";

# MARS home directory.
my $HOME_FOLDER = "/usr/local/megha";
#my $HOME_FOLDER = "/usr/local/megha";

# This script will use this folder to do work.
my $SCRIPT_ROOT_FOLDER = "";

# Raw data directory.
my $RAW_DATA_FOLDER = "";

# All processed data will be zipped and stored in this directory.
my $BACKUP_FOLDER = "";

# All the MAPS will be dumped here.
my $HASH_FOLDER = "";

# This file is used to keep entries of target machines with their last configuration data creation time.
my $HOST_LIST_FILE  = "";

# Hash file of old run.
my $HASH_FILE = "";

# Actual raw processing is done here.
my $WORK_FOLDER = "";

# Perf data will be kept here.
my $PERF_FOLDER = "";

# Conf data will be kept here.
my $CONF_FOLDER  = "";

# db/probe/data folder.
my $DEST_FOLDER = "";

# Aggregated performance data is stored here.
my $AGG_FOLDER = "";

# Site name or customer name.
my $siteName = "";

# Probe id.
my $probeId = "";

# db/probe directory.
my $DB_PROBE_DIR = "";

# conf/probe directory.
my $CONF_PROBE_DIR = "";

# Probe home directory.
my $PROBE_DIR = "";

# Conf final directory.
my $CONF_FINAL_DIR = "";

# Interval in seconds (1 hour) between 2 sets of configuration data collection.
my $CONF_DATA_INTERVAL = 3600;

# Indicate maximum number of files which we need to process in this process.
my $MAX_FILE_FOR_PROCESSING = 100;

# This map variable is used to store aggregated metric data.
my %aggregateResource = ();

# Used to store configuration information for discovered resources.
my %resources = ();

# This map variable is used to store path of aggregated folder for each type of resource.
my %aggPerfFolder = ();

# Used to store time-stamp for configuration data generation.
my $confTimestamp = "";

# It is used to decide whether we need to aggregate data for a minute.
my $aggrMinuteData = 0;

# This variable set to 1 if there is need to generate configuration data.
my $genConfigData = 0;

# Used to store Probe specific property file name.
my $propFileName = "";

# Used to store the log file name.
my $logFile;

# Log file handle.
my $LOG_HANDLE;

# Store Probe name.
my $probeName = "";

# Flush output after every write. Unbuffered output is usually desired.
$| = 1;

# Initialize the environment.
my $status = init();
if (0 != $status)
{
    $logObj->error("Call to init() failed.");
    goto EXIT;
}

# Scan new files.
$status = scanFiles();
if (0 != $status)
{
    $logObj->error("Call to scanFiles() failed.");
    goto EXIT;
}

# Generate MARS files from the data.
$status = generateOutput();
if (0 != $status)
{
    $logObj->error("Call to generateOutput() failed.");
    goto EXIT;
}

# Dump aggregated resource information.
$status = dumpAggrResources();
if (0 != $status)
{
    $logObj->error("Call to dumpAggrResources() failed.");
    goto EXIT;
}

EXIT:

exit $status;

# This function is used to initialize the environment to run the Perl script correctly.
#
# @affected global variables :
#   $propFileName
#   $logFile
#
# @return:
#   0 if Success
#   1 if Error
sub init
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # Probe home directory.
    $DB_PROBE_DIR = "$HOME_FOLDER/db/probe";

    # Conf probe directory.
    $CONF_PROBE_DIR = "$HOME_FOLDER/conf/probe";

    # This variable stores Log folder path.
    my $logFolder = $HOME_FOLDER."/logs";

    # If log folder does not exist then create it.
    if (! (-e $logFolder))
    {
        mkpath($logFolder);
    }

    # Get commonLogModule instance.
    $logObj = commonLogModule->getInstance($logFolder, $probeType, $probeName);

    # If number of command line arguments are not correct then quit.
    if (1 != ($#ARGV + 1))
    {
        print("perl processRawData.pl <Property File name>");

        $logObj->error("Number of arguments to processRawData.pl are not correct.");
        $logObj->error("perl processRawData.pl <Property File Path>.");
        $logObj->error("Input arguments = ".($#ARGV + 1).".");

        $retVal = 1;
        goto EXIT;
    }

    # Get the property file name from command line argument.
    $propFileName = $ARGV[0];

    # Check instance of this script corresponding to property file.
    $status = checkOtherInstanceRunning();
    if (0 != $status)
    {
        $logObj->warn("Other instance for property file [$propFileName] is also running.");
        $retVal = 1;

        goto EXIT;
    }

    # Read the .properties file corresponding to Linux Probe.
    $status = readPropertyFile();
    if (0 != $status)
    {
        $logObj->error("Call to readPropertyFile() failed.");
        $retVal = 1;

        goto EXIT;
    }

    # Update the final log message information with populated Linux probe id.
    $logObj->setProbeId($probeName);

    # Reload any previous hash.
    if (-e $HASH_FILE)
    {
        open(HASH_HANDLE, '<', $HASH_FILE) or die $logObj->error("Failed to open file : [$HASH_FILE], error [$!].");

        my $var = "";

        while (<HASH_HANDLE>)
        {
            chomp;
            $var .= $_;
        }

        # Here $var contain a Perl expression which set $VAR1 (dumper method create it) with a hash table. "eval" function will set $VAR1 variable
        # with the hash table.
        my $VAR1;
        eval $var;

        %aggregateResource = %$VAR1;

        close(HASH_HANDLE);
    }

    # Probe workspace.
    $PROBE_DIR = "$HOME_FOLDER/db/probe/$siteName/$probeId";

    # This script will use this folder to do work.
    $SCRIPT_ROOT_FOLDER = "$PROBE_DIR/work";

    # Raw data directory.
    $RAW_DATA_FOLDER = "$PROBE_DIR/rawdata";

    # Create RAW_DATA_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($RAW_DATA_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$RAW_DATA_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # All the processed data will be zipped and stored in this directory.
    $BACKUP_FOLDER = "$PROBE_DIR/backup";

    # Create the backup folder.
    unless (defined eval {mkpath($BACKUP_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$BACKUP_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # All the MAPS will be dumped here.
    $HASH_FOLDER = "$SCRIPT_ROOT_FOLDER/aggrStaging";

    # Create HASH_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($HASH_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$HASH_FOLDER].");
        $retVal = 1;
        goto EXIT;
    }

    # This file is used to keep entries of target machines with there last configuration creation time.
    $HOST_LIST_FILE = "$SCRIPT_ROOT_FOLDER/hostlist.config";

    # Hash file of old run.
    $HASH_FILE = "$HASH_FOLDER/imdthash.hash";

    # Actual raw processing is done here.
    $WORK_FOLDER = "$SCRIPT_ROOT_FOLDER/partialMarsData";

    # Create WORK_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($WORK_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$WORK_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # Perf data will be kept here.
    $PERF_FOLDER = "$WORK_FOLDER/perf";

    # Create PERF_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($PERF_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$PERF_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # Conf data will be kept here.
    $CONF_FOLDER = "$WORK_FOLDER/conf";

    # Create CONF_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($CONF_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$CONF_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # Final configuration and performance files will be kept here.
    $DEST_FOLDER = "$HOME_FOLDER/db/probe/data";

    # Aggregated performance data is stored here.
    $AGG_FOLDER  = "$PERF_FOLDER/agg";

    # Create AGG_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($AGG_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$AGG_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

EXIT:

    return $retVal;
}

# It looks for raw data files available at specific location and process them. It keeps collected configuration data in memory while generate
# files for storing performance data.
#
# @param:
#    Nothing.
#
# @return:
#    0 if Success
#    1 if Error
sub scanFiles
{
    # Get the list of files in "rawdata" folder.
    my @files = <$RAW_DATA_FOLDER/*>;

    # Used to store actual file names if file is zipped.
    my $actualFile = "";

    # Number of files which we need to process max.
    my $fileLimitCount = $MAX_FILE_FOR_PROCESSING;

    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # If no file available for processing then exit.
    if ($#files < 0)
    {
        $logObj->warn("No raw files in the directory: [$RAW_DATA_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # Create DEST_FOLDER directory, if does not exist.
    unless (defined eval {mkpath($DEST_FOLDER)})
    {
        $logObj->error("Not able to create folder: [$DEST_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # Process each file.
    foreach my $file (@files)
    {
        # If processed maximum number of files then exit.
        if ($fileLimitCount == 0)
        {
            $logObj->info("Processeed $MAX_FILE_FOR_PROCESSING files, so exiting.");
            $retVal = 0;

            goto EXIT;
        }

        # tmp* files are temporary created by rt_recv script which renames it at the end. So don't process them.
        if ($file =~ m/^tmp./)
        {
            next;
        }

        # Process only plain text file.
        if (! -f $file)
        {
            $logObj->warn("Found non -plain text file: [$file]");
            next;
        }

        # Process only non-zero size file.
        if (-s $file == 0)
        {
            # Remove files having size zero.
            if (executeCommand("rm -f $file") != 0)
            {
                $logObj->warn("Removing zero size file failed: [$file].");
            }

            next;
        }

        $fileLimitCount--;

        # Get file extension.
        my $ext = (fileparse($file, qr/\.[^.]*/))[2];

        # If current file is .gz file then unzip the file.
        if ($ext eq ".gz")
        {
            if (executeCommand("gunzip -f ".$file) != 0)
            {
                $logObj->warn("System call gunzip failed for file: [$file].");
                next;
            }

            $actualFile = $file;
            $actualFile =~ s/\.gz$//;
        }
        else
        {
            $actualFile = $file;
        }

        # Process the raw data and backup raw files.
        if (processFile($actualFile, \$fileLimitCount) == 0)
        {
            my $backUpDir = getBackUpFolder($actualFile);

            if (defined($backUpDir)) {
                # Zip the file and move to backup folder.
                backupFile($backUpDir, $actualFile);

                $logObj->debug("Backup directory [$backUpDir], actual file [$actualFile].");
            }
        }
        else
        {
            # Not returning from here so that we can process remaining files.
            $logObj->error("Call to processFile() failed.");
        }
    }

EXIT:

    return $retVal;
}

# This function does actual Raw file processing.
#
# @param:
#    $_[0]: [In] File which needs to be processed.
#    $_[1]: [In/Out] Number of files which we can still process.
#
# @return:
#   0 if Success
#   1 if Error
sub processFile
{
    my ($fileName, $fileCount) = @_;

    # Used to store the time for next available data.
    my $timeStr = "";

    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    # Get the file name from complete path.
    my $temp = basename($fileName);

    # Strip off leading localhost_ or localhost.localdoamin_.
    $temp =~ s/^localhost_//g;
    $temp =~ s/^localhost\.localdomain_//g;

    my ($hostName, $ymd, $hms, $os, $type, $interval, $samples);

    # File name looks like 192.168.159.190_20140714_100404_linux_iostat_1_60_cumulus_local.
    ($hostName, $ymd, $hms, $os, $type, $interval, $samples) = split(/_/, $temp);

    # File was not in correct format so we did not process it.
    if ($hostName eq "" || $ymd eq "" || $hms eq "" || $os eq "" || $type eq "" || $interval eq "" || $samples eq "")
    {
        $logObj->warn("File name not in proper format: [$fileName].");

        # Increment the file count as we are not processing this file.
        ${$fileCount}++;

        $retVal = 1;
        goto EXIT;
    }

    # Check whether configuration file should be created or not.
    $genConfigData =  $genConfigData || isGenerateConfig($hostName, $CONF_DATA_INTERVAL);

    $logObj->debug("Data type for current raw file: [$type].");

    if ($type eq "conf")
    {
        if (saveConfFile($fileName, $hostName) != 0)
        {
            $logObj->error("Call to function saveConfFile() failed.");
            $retVal = 1;

            goto EXIT;
        }
    }

    # Set host resource configuration data.
    $resources{$HOST_PREFIX."#".$hostName}{"name"} = $hostName;
    $resources{$HOST_PREFIX."#".$hostName}{"relations"} = "";

    # If there is need to generate configuration data and we haven't set the time for this data then set it. With this condition, we will use the
    # time-stamp where configuration data collection is needed.
    # to set the time again with same value.
    if ($confTimestamp eq "" && $genConfigData == 1)
    {
        $confTimestamp = $ymd."_".$hms;
    }

    # Check every time so that we gather only minute level if enabled. Ideally this condition is useful only when we have changed the interval for
    # a remote host dynamically. In that case, this script can find files with different intervals. So we need to make sure that we are not enabling
    # aggregate where it is not required.
    $aggrMinuteData = 0;

    if ($interval < 60)
    {
        # We want to perform minute level averaging of data since we are collecting data with interval in seconds.
        $logObj->info("Enabling minute level averaging as interval is [$interval] seconds.");
        $aggrMinuteData = 1;

        $timeStr = adjustDataBoundary($ymd, $hms, $interval, 0);
    }
    else
    {
        $logObj->info("No need of minute level averaging as interval is [$interval] seconds.");
        $timeStr = "${ymd}_${hms}";
    }

    # Create folder for storing the performance data.
    my $perfFolder = "$PERF_FOLDER/"."$timeStr";
    unless (defined eval {mkpath($perfFolder)})
    {
        $logObj->error("Not able to create folder: [$perfFolder].");
        $retVal = 1;

        goto EXIT;
    }

    # If there is need to aggregate the data then set the path for it.
    if ($aggrMinuteData == 1)
    {
        if (!$aggPerfFolder{$type})
        {
            $aggPerfFolder{$type} = $AGG_FOLDER."/".$timeStr;
        }
    }

    if ($type eq "mpstat")
    {
        if (processLinuxMpstat($hostName, $perfFolder, $aggPerfFolder{$type}, $interval, $samples, $fileName) != 0)
        {
            $logObj->error("Call to function processLinuxMpstat failed.");
            $retVal = 1;

            goto EXIT;
        }
    }
    elsif ($type eq "vmstat")
    {
        if (processLinuxVmstat($hostName, $perfFolder, $aggPerfFolder{$type}, $interval, $samples, $fileName) != 0)
        {
            $logObj->error("Call to function processLinuxVmstat failed.");
            $retVal = 1;

            goto EXIT;
        }
    }
    elsif ($type eq "iostat")
    {
        if (processLinuxIostat($hostName, $perfFolder, $aggPerfFolder{$type}, $interval, $samples, $fileName) != 0)
        {
            $logObj->error("Call to function processLinuxIostat failed.");
            $retVal = 1;

            goto EXIT;
        }
    }
    elsif ($type eq "netstat")
    {
        if (processLinuxNetstat($hostName, $perfFolder, $aggPerfFolder{$type}, $interval, $samples, $fileName) != 0)
        {
            $logObj->error("Call to function processLinuxNetstat failed.");
            $retVal = 1;

            goto EXIT;
        }
    }

EXIT:

    return $retVal;
}

# Generate final configuration and performance files at specific location.
#
# @param:
#    None.
#
# @return:
#    0 if success.
#    1 if failure.
sub generateOutput
{
    # This variable is used to store the return value of function calls.
    my $retVal = 0;

    if ($confTimestamp != "" && $genConfigData == 1)
    {
        # Generate final configuration file.
        if (generateFinalConfFiles() != 0)
        {
            $logObj->error("Call to generateFinalConfFiles() failed.");
            $retVal = 1;

            goto EXIT;
        }

        # Reset it for the next set of files.
        $genConfigData = 0;
    }

    # Generate final performance files.
    if (generateFinalPerfFiles() != 0)
    {
        $logObj->error("Call to generateFinalPerfFiles() failed.");
        $retVal = 1;

        goto EXIT;
    }

EXIT:

    # Change current directory.
    chdir($SCRIPT_ROOT_FOLDER);

    return $retVal;
}

# Generate final MARS configuration file and place it at proper location.
#
# @param:
#    None.
#
# @return:
#    0 if success.
#     1 if failure.
sub generateFinalConfFiles
{
    my $retVal = 0;

    # If destination folder for final configuration file not exist then it is an error.
    if (!(-e $CONF_FOLDER))
    {
        $logObj->error("Folder does not exist: [$CONF_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    # Set final configuration file path.
    my $snapshotFileName = "Conf_${confTimestamp}_${siteName}_minisnapshot_Linux.txt" ;
    my $snapshotFilePath = "$CONF_FOLDER/$snapshotFileName";

    # Open snapshot file.
    if (!open(FILEC, "> $snapshotFilePath"))
    {
        $logObj->error("Unable to open file [$snapshotFilePath] for writing.");
        $retVal = 1;

        goto EXIT;
    }

    # Put meta-data information in file.
    print(FILEC "%meta\n");
    print(FILEC "probe_id:$siteName" . "_" . "$probeName\n");
    print(FILEC "probe_type:Linux\n");
    print(FILEC "probe_site:$siteName\n");
    print(FILEC "probe_server:MARS\n");
    print(FILEC "probe_version:10\n");
    print(FILEC "%meta\n");
    print(FILEC "t:$confTimestamp\n");

    # Parse content of hostConfigData.txt file.
    if (readHostConfigData() != 0)
    {
         $logObj->warn("Failed to read host configuration data file.");
    }

    # Put resources and their relation information in the file.
    for my $resource (keys(%resources))
    {
        print(FILEC "R:$resource\n");
        print(FILEC "a:name=".$resources{$resource}{"name"}."\n");

        $logObj->debug("Configuration resource signature: [$resource].");

        # Adding configuration attribute related to current resource.
        if (exists $confAttributeMap{$resource})
        {
            print(FILEC $confAttributeMap{$resource});
        }

        if ($resources{$resource}{"relations"} ne "")
        {
            print(FILEC "r:".$resources{$resource}{"relations"}."\n");
        }
    }

    close(FILEC);

    # Set path where we need to put snapshot file. Data collection framework will pick this file from there.
    $CONF_FINAL_DIR = $DB_PROBE_DIR . "/" . $siteName . "/" . $probeId . "/" . "conf";
    
    # Create CONF_FINAL_DIR directory, if does not exist.
    unless (defined eval {mkpath($CONF_FINAL_DIR)})
    {
        $logObj->error("Not able to create folder: [$CONF_FINAL_DIR].");
        $retVal = 1;

        goto EXIT;
    }
    
    # Move snapshot file.
    if (executeCommand("mv $snapshotFilePath $CONF_FINAL_DIR") != 0)
    {
        $logObj->error("Failed to move file [$snapshotFilePath] to directory [$CONF_FINAL_DIR].");
        $retVal = 1;

        goto EXIT;
    }
    else
    {
        # Rename file with extension as .DONE so that data collection framework will pick for further processing.
        rename($CONF_FINAL_DIR."/".$snapshotFileName, $CONF_FINAL_DIR."/".$snapshotFileName.".DONE");
    }

EXIT:

    return $retVal;
}

# Move all Perf related files to final destination folder.
#
# @param:
#    None.
#
# @return:
#    0 if success.
#    1 if failure.
sub generateFinalPerfFiles
{
    my $retVal = 0;

    # First handle aggregated perf files.
    if ($aggrMinuteData == 1)
    {
        chdir($AGG_FOLDER);

        # Find files in aggr folder with "_" in the file name.
        my @files = <*_*>;

        if ($#files < 0)
        {
            $logObj->error("[$AGG_FOLDER] folder is empty.");
            $retVal = 1;

            goto EXIT;
        }

        # Move all the aggregated files to perf directory.
        foreach my $file (@files)
        {
            # Set complete path for aggregated files.
            my $aggrFilePath = $DB_PROBE_DIR."/".$siteName."/".$probeId."/perf/Perf_".$file."_MinAgg";

            # Move file to final location.
            if (executeCommand("mv $file $aggrFilePath") != 0)
            {
                $logObj->error("Falied to move file [$file] to destination [$aggrFilePath].");
                $retVal = 1;

                goto EXIT;
            }
            else
            {
                # Rename file with extension as .DONE so that data collection framework will pick it and will do further processing.
                rename($aggrFilePath, $aggrFilePath.".DONE");
            }
        }
    }

    # Delete aggregated folder as we processed all files from there.
    if (-e $AGG_FOLDER)
    {
        $logObj->info("Removing aggregate folder [$AGG_FOLDER].");
        if (executeCommand("rm -rf $AGG_FOLDER") != 0)
        {
            $logObj->error("Removing aggregate folder failed. Path: [$AGG_FOLDER].");
        }
    }

    # If final perf folder not exist then error out.
    if (! (-e $PERF_FOLDER))
    {
        $logObj->error("Folder not exist: [$PERF_FOLDER].");
        $retVal = 1;

        goto EXIT;
    }

    chdir($PERF_FOLDER);

    # Find all files in the current folder with name having "_".
    my @files = <*_*>;

    if ($#files < 0)
    {
        $logObj->error("Folder [$PERF_FOLDER] is empty.");
        $retVal = 1;

        goto EXIT;
    }

    # Move all perf files to final perf directory.
    foreach my $file (@files)
    {
        my $filePath = $DB_PROBE_DIR."/".$siteName."/".$probeId."/perf/Perf_".$file;

        # Move files.
        if (executeCommand("mv $file $filePath") != 0)
        {
            $logObj->error("Falied to move file [$file] to destination [$filePath].");
            $retVal = 1;

            goto EXIT;
        }
        # Rename file with extension as .DONE so that data collection framework will pick it and will do further processing.
        else
        {
            rename($filePath, $filePath.".DONE");
        }
    }

EXIT:

    return $retVal;
}

# This function will create the archive folder.
#
# @param:
#    $_[0]: [In] Backup folder.
#    $_[1]: [In] File to backup.
#
# @return:
#    Nothing.
sub backupFile
{
    my ($backupDir, $fileName) = @_;

    my ($fName, $dirName) = fileparse($fileName);

    # Set zip file name.
    my $zipName = $backupDir."/".basename($fileName).".gz";

    $logObj->debug("Creating zip file: [$zipName].");

    # Save current working directory.
    my $cwd = cwd();

    # Change directory to the folder where file is present.
    chdir($dirName);

    # If file size is 0 then no need to create backup file for it.
    if (executeCommand("gzip -c -q -r $fileName > $zipName") != 0)
    {
        # Not able to create backup for it.
        $logObj->warn("Zip operation failed for file: [$fileName].");
    }

    # Delete file as we have backup now.
    if (!unlink($fileName))
    {
        $logObj->warn("Failed to delete file: [$fileName].");
    }

    # Go back to previous directory.
    chdir($cwd);
}

# Create backup folders based on one hour interval.
#
# @param:
#    $_[0]: [In] File name.
#
# @return:
#    If success - Backup directory name.
#    If failure - undef
sub getBackUpFolder
{
    my ($fileName) = @_;

    my $temp = basename($fileName);

    # Remove localhost part from the file name.
    $temp =~ s/^localhost_//g;
    $temp =~ s/^localhost\.localdomain_//g;

    my ($hostName, $ymd, $hms, $os, $type, $interval, $samples) = split(/_/, $temp);

    my $hour = substr($hms, 0, 2);

    my $bkupDir = $BACKUP_FOLDER."/".$ymd."/".$hour;

    unless (defined eval {mkpath($bkupDir)})
    {
        $logObj->warn("Failed to create backup folder: [$bkupDir].");

        return undef;
    }

    return $bkupDir;
}

# This function reads each <host IP>_hostConfigData.txt file and load its entries in the map.
#
# @param: None
#
# @return:
#    0 if success
#    1 if failure
sub readHostConfigData()
{
    my $retVal = 0;

    # Get the list of files in temporary configuration folder.
    my @files = <$CONF_FOLDER/*$TEMP_CONF_FILE_NAME>;

    $logObj->debug("Total configuration files found: [".($#files + 1)."].");

    if ($#files < 0)
    {
         $logObj->debug("No files found in the directory. Path: [$CONF_FOLDER].");
         $retVal = 1;

         goto EXIT;
    }

    # Read each file.
    foreach my $file (@files)
    {
        if (! -f $file)
        {
            $logObj->warn("Found non-plain text file: [$file]. Ignoring it.");
            next;
        }

        $logObj->debug("Reading file: [$file].");

        # Open the file.
        if (!open(FILE, $file))
        {
            $logObj->error("Unable to open file: [$file]. Error: [$!].");
            $retVal = 1;

            goto EXIT;
        }

        while (<FILE>)
        {
            # Remove the new line character.
            $_ =~ s/\r\n//g;
            $_ =~ s/\n//g;

            # Strip of any leading space
            s/^\s+//;

            # Ignore empty lines.
            if (/^$/)
            {
                next;
            }

            # Line format: <MARS resource signature>_<configuration attribute name>=<configuration attribute value>
            # Split the string with "_".
            my @items = split('_', $_);
            if ($#items != 1)
            {
                $logObj->error("Invalid configuration information found. Info: [$_].");
                next;
            }

            $logObj->debug("Conf attribute found. Key: [$items[0]]. Value: [$items[1]].");

            # Add configuration information in the global map. Stores value in the proper MARS format. Eg: a:<key>=<value>
            my $attrValue = "a:".$items[1]."\n";
            $confAttributeMap{$items[0]} = $attrValue;

            $logObj->debug("Added configuration attribute to the Map. Attribute: [$attrValue]");
        }

        # Close file handle.
        close(FILE);
    }

EXIT:

    return $retVal;
}

# This function copies raw configuration data file to a temp location with the name hostConfigData.txt file.
#
# @param:
#   $_[0]: File which needs to be copied.
#   $_[1]: Host name.
#
# @return:
#    0 if success
#    1 if failure
sub saveConfFile
{
    $logObj->debug("Entered saveConfFile().");

    my ($confFile, $hostName) = @_;

    my $retVal = 0;

    # Set temporary configuration file path with file name as: <HostName>_hostConfigData.txt
    my $tempConfigDataFile = $CONF_FOLDER."/".$hostName."_".$TEMP_CONF_FILE_NAME;

    # Delete old hostConfigData file and create the new one with updated values.
    if (-e $tempConfigDataFile)
    {
        # Delete file.
        if (executeCommand("rm -f $tempConfigDataFile") != 0)
        {
            $logObj->error("Failed to delete the file. File: [$tempConfigDataFile].");
            $retVal = 1;

            goto EXIT;
        }
        else
        {
            $logObj->debug("Deleted file: [$tempConfigDataFile].");
        }
    }

    # Move the current raw configuration data file to hostConfigData.txt file.
    if (executeCommand("cp $confFile $tempConfigDataFile") != 0)
    {
        $logObj->error("Failed to delete the file. File: [$tempConfigDataFile].");
        $retVal = 1;

        goto EXIT;
    }
    else
    {
        $logObj->debug("Moved raw configuration data file [$confFile] to [$tempConfigDataFile].");
    }

EXIT:

    return $retVal;
}

# Process mpstat command output and dump it in physical file.
#
# @param:
#    $_[0]: [In] Host name.
#    $_[1]: [In] Destination folder for Perf data.
#    $_[2]: [In] Destination folder for Aggregate Perf data.
#    $_[3]: [In] Data collection interval.
#    $_[4]: [In] Number of samples.
#    $_[5]: [In] Raw data file name.
#
# @return:
#    0 on success.
#    1 on failure.
sub processLinuxMpstat
{
    my ($hostName, $folder, $aggPerfFolder, $interval, $samples, $fileName) = @_;

    # Used to store return value of this function.
    my $retVal = 0;

    # Used to store CPU index value.
    my $cpuColumnIndex = -1;

    # Used to store information about header in mpstat command output.
    my @headerCols = ();

    # Used to store information about data in mpstat command output.
    my @dataCols = ();

    # Used to store Host perf metrics. It is used as <counter name, <resource name, performance data>>.
    my %hostMetrics = ();

    # Used to store Host CPU perf metrics. It is used as <counter name, <resource name, performance data>>.
    my %cpuMetrics = ();

    if (!open(FILE, $fileName))
    {
        $logObj->error("Unable to open file [$fileName]. Error: [$!].");
        $retVal = 1;

        goto EXIT;
    }

    $logObj->info("Processing file: " . $fileName);

    # Create folder for Host resource.
    unless (defined eval {mkpath("$folder/$HOST_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_PREFIX].");
        $retVal = 1;

        goto EXIT;
    }

    # Create folder for Host CPU resource.
    unless (defined eval {mkpath("$folder/$HOST_CPU_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_CPU_PREFIX].");

        $retVal = 1;
        goto EXIT;
    }

    while (<FILE>)
    {
        # Remove the new line character.
        $_ =~ s/\r\n//g;
        $_ =~ s/\n//g;

        # This is the first line e.g. Linux 2.6.18-92.el5 (pollux)  03/03/12. So ignore it.
        if (/Linux/)
        {
            next;
        }

        # Ignore empty lines.
        if (/^$/)
        {
            next;
        }

        # There are Average values at end. Ignore those as we don't need them.
        if (/average/i)
        {
            next;
        }

        # This is header line like
        # 18:59:02   CPU   %user   %nice  %sys %iowait  %irq   %soft  %steal   %idle  intr/s
        if (/CPU/)
        {
            # We are assuming that the metrics will vary based on Linux kernel. So the number may be different but the names will be fixed.
            @headerCols = split(/\s+/, $_);

            # Check CPU column index for following types of headers -
            # 01:00:01 AM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle
            # Or
            # 04:54:07     CPU   %user   %nice    %sys %iowait    %irq   %soft  %steal   %idle    intr/s
            for (my $i = 0; $i <= $#headerCols; $i++)
            {
                if ($headerCols[$i] eq "CPU")
                {
                    $cpuColumnIndex = $i;
                    $logObj->debug("CPU column index is: [$cpuColumnIndex].");

                    last;
                }
            }

            if ($cpuColumnIndex == -1)
            {
                $logObj->error("CPU Header not found. Unexpected mpstat file: [$fileName]");
                $retVal = 1;

                goto EXIT;
            }

            $logObj->debug("Number of header columns found: [$#headerCols].");
        }
        else
        {
            # Data line.
            @dataCols = split(/\s+/, $_);

            if ($#dataCols < 0)
            {
                next;
            }

            # The number of header and data columns must match otherwise we can't correlate the values.
            if ($#headerCols != $#dataCols)
            {
                $logObj->debug("Data does not match the header columns: [$#headerCols], [$#dataCols].");

                next;
            }

            if ($dataCols[$cpuColumnIndex] eq "all")
            {
                # This is host level data.
                for (my $i = ($cpuColumnIndex + 1); $i <= $#dataCols; $i++)
                {
                    $hostMetrics{$headerCols[$i]}{$hostName} .= $dataCols[$i].",";
                }
            }
            else
            {
                # Individual CPU metric
                for (my $i = ($cpuColumnIndex + 1); $i <= $#dataCols; $i++)
                {
                    $cpuMetrics{$headerCols[$i]}{$hostName."-".$dataCols[$cpuColumnIndex]} .= $dataCols[$i].",";
                }

                # Add CPU as resource with name and relation information.
                $resources{$HOST_CPU_PREFIX."#".$hostName."-".$dataCols[$cpuColumnIndex]}{"name"} = $hostName."-".$dataCols[$cpuColumnIndex];
                $resources{$HOST_CPU_PREFIX."#".$hostName."-".$dataCols[$cpuColumnIndex]}{"relations"} = $HOST_PREFIX."#".$hostName;
            }
        }
    }

    # Dump collected data in physical file.
    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_PREFIX, $interval, $samples, \%hostMetrics, \%MPSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics() failed for Host.");
        $retVal = 1;

        goto EXIT;
    }

    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_CPU_PREFIX, $interval, $samples, \%cpuMetrics, \%MPSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics() failed for CPU.");
        $retVal = 1;

        goto EXIT;
    }

EXIT:

    return $retVal;
}

# Process vmstat command output and dump it in physical file.
#
# @param:
#    $_[0]: [In] Host name.
#    $_[1]: [In] Destination folder for Perf data.
#    $_[2]: [In] Destination folder for Aggregate Perf data.
#    $_[3]: [In] Data collection interval.
#    $_[4]: [In] Number of samples.
#    $_[5]: [In] Raw data file name.
#
# @return:
#    0 on success.
#    1 on failure.
sub processLinuxVmstat
{
    my ($hostName, $folder, $aggPerfFolder, $interval, $samples, $fileName) = @_;

    my $retVal = 0;

    # Used to store information about header in vmstat command output.
    my @headerCols = ();

    # Used to store information about data in vmstat command output.
    my @dataCols = ();

    # Used to store Host perf metrics. It is used as <counter name, <resource name, performance data>>.
    my %hostMetrics = ();

    # If 0 then it means that we have read the first line of "data" part.
    my $first = 1;

    if (!open(FILE, $fileName))
    {
        $logObj->error("Unable to open file [$fileName], error: [$!].");
        $retVal = 1;

        goto EXIT;
    }

    $logObj->info("Processing file: [$fileName].");

    # Create folder for Host resource.
    unless (defined eval {mkpath("$folder/$HOST_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_PREFIX].");
        $retVal = 1;

        goto EXIT;
    }

    while (<FILE>)
    {
        # Remove the new line character.
        $_ =~ s/\r\n//g;
        $_ =~ s/\n//g;

        # Strip of any leading space
        s/^\s+//;

        # This is the first line like "procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------". So ignore it.
        if (/procs/)
        {
            next;
        }

        # Ignore empty lines.
        if (/^$/)
        {
            next;
        }

        # This is the header line like
        # r  b   swpd   free  inact active   si   so  bi  bo   in   cs us sy id wa st
        if (/swpd/ || /si/ || /bi/)
        {
            # We are assuming that the metrics will vary based on Linux kernel. So the number may be different but the names will be fixed.
            @headerCols = split(/\s+/, $_);

            $logObj->debug("Number of header columns found: [$#headerCols].");
        }
        else
        {
            # Ignore the first line as it is value from boot.
            if ($first == 1)
            {
                $first = 0;
                next;
            }

            # Data line
            @dataCols = split(/\s+/, $_);

            if ($#dataCols < 0)
            {
                next;
            }

            # The number of header and data columns must match otherwise we can't correlate the values.
            if ($#headerCols != $#dataCols)
            {
                $logObj->debug("Data does not match the header columns: [$#headerCols], [$#dataCols].");

                next;
            }

            # Put all metrics at host level.
            for (my $i = 0; $i <= $#dataCols; $i++)
            {
                $hostMetrics{$headerCols[$i]}{$hostName} .= $dataCols[$i].",";
            }
        }
    }

    # Dump collected data in physical file.
    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_PREFIX, $interval, $samples, \%hostMetrics, \%VMSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics failed.");
        $retVal = 1;
    }

EXIT:

    return $retVal;
}

# Process iostat command output and dump it in physical file.
#
# @param:
#    $_[0]: [In] Host name.
#    $_[1]: [In] Destination folder for Perf data.
#    $_[2]: [In] Destination folder for Aggregate Perf data.
#    $_[3]: [In] Data collection interval.
#    $_[4]: [In] Number of samples.
#    $_[5]: [In] Raw data file name.
#
# @return:
#    1 on success.
#    0 on failure.
sub processLinuxIostat
{
    my ($hostName, $folder, $aggPerfFolder, $interval, $samples, $fileName) = @_;

    my $retVal = 0;

    # Used to store information about header in iostat command output.
    my @headerCols = ();

    # Used to store information about data in iostat command output.
    my @dataCols = ();

    # Used to store Disk metrics. It is used as <counter name, <resource name, performance data>>.
    my %diskMetrics = ();

    # Used to store Host metrics. It is used as <counter name, <resource name, performance data>>.
    my %hostMetrics = ();

    # If 1 then it means that we are about to read the first line of "data" part.
    my $first = 0;

    # Used to store host aggregated data.
    my %hostAgg = ();

    if (!open(FILE, $fileName))
    {
        $logObj->error("Unable to open file [$fileName], error: [$!]");
        $retVal = 0;

        goto EXIT;
    }

    $logObj->info("Processing file: [$fileName].");

    # Create folder for Host resource.
    unless (defined eval {mkpath("$folder/$HOST_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_PREFIX].");
        $retVal = 1;

        goto EXIT;
    }

    # Create folder for Host Disk resource.
    unless (defined eval {mkpath("$folder/$HOST_DISK_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_DISK_PREFIX].");
        $retVal = 1;

        goto EXIT;
    }

    # Used to skip next line if set to 1.
    my $skipNextLine = undef;

    # Used to store number of data set which we are processing.
    my $recordCount = 0;

    while (<FILE>)
    {
        # Remove the new line character.
        $_ =~ s/\r\n//g;
        $_ =~ s/\n//g;

        # Strip of any leading space.
        s/^\s+//;

        # This is the first line e.g. Linux 2.6.18-92.el5 (pollux)  03/03/12. So ignore it.
        if (/Linux/)
        {
            next;
        }

        # Ignore time-stamp line like "Wed Apr 11 16:04:35 CEST 2012"
        if (/\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}/)
        {
            next;
        }

        # Ignore time lines like Time: 00:30:01.
        if (/Time/)
        {
            next;
        }

        # Ignore empty lines.
        if (/^$/)
        {
            next;
        }

        # Ignore any other metric like avg-cpu
        # avg-cpu:  %user   %nice %system %iowait  %steal   %idle
        # 2.60  0.01  0.44  3.01  0.00   93.93
        if (/avg-cpu/)
        {
            $skipNextLine = 1 ;
            next;
        }

        if ($skipNextLine)
        {
            $skipNextLine = undef;
            next;
        }

        # This is header line.
        # Device:     rrqm/s   wrqm/s   r/s   w/s  rMB/s  wMB/s avgrq-sz avgqu-sz   await  svctm  %util
        if (/Device/)
        {
            # We are assuming that the metrics will vary based on Linux kernel. So the number may be different but the names will be fixed.
            @headerCols = split(/\s+/, $_);

            $logObj->debug("Number of header columns found: [$#headerCols].");

            $first = 1;
            $recordCount++;
        }
        # This is data line.
        else
        {
            # If we are starting data processing for current time slot.
            if ($first == 1)
            {
                # If we already collected data for previous time slot then we can aggregate that data.
                if ($recordCount > 2)
                {
                    aggregateIostatParams(\%hostAgg, \%hostMetrics, $hostName);
                }

                %hostAgg = ();
                $first = 0;
            }

            # Data line
            @dataCols = split(/\s+/, $_);

            if ($#dataCols < 0)
            {
                next;
            }

            # The number of header and data columns must match otherwise we can't correlate the values.
            if ($#headerCols != $#dataCols)
            {
                $logObj->debug("Data does not match the header columns: [$#headerCols], Data columns [$#dataCols].");

                next;
            }

            # Support only raw disks like fio, sd, and hd.
            if (!($dataCols[0] =~ /fio[a-z]+$/) && !($dataCols[0] =~ /sd[a-z]+$/) && !($dataCols[0] =~ /hd[a-z]+$/))
            {
                next;
            }

            # As first time slot data is for boot, we are ignoring it.
            if ($recordCount > 1)
            {
                # Get individual disk metric.
                for (my $i = 1; $i <= $#dataCols; $i++)
                {
                    # Convert MB/s to KB/s.
                    if ($headerCols[$i] =~ /MB/)
                    {
                        $dataCols[$i] = $dataCols[$i] * 1024;
                    }

                    # Roll up r/w ps and r/w kbps to host level.
                    if ($headerCols[$i] =~ /[rw]\/s/ || $headerCols[$i] =~ /[rw]kB\/s/ )
                    {
                        $hostAgg{$headerCols[$i]} += $dataCols[$i];
                    }

                    # Update disk level metrics.
                    $diskMetrics{$headerCols[$i]}{$hostName."-".$dataCols[0]} .= $dataCols[$i].",";
                }
            }

            # Add Disk as resource
            $resources{$HOST_DISK_PREFIX."#".$hostName."-".$dataCols[0]}{"name"} = $hostName."-".$dataCols[0];
            $resources{$HOST_DISK_PREFIX."#".$hostName."-".$dataCols[0]}{"relations"} = $HOST_PREFIX."#".$hostName;
        }
    }

    # This is to aggregate the data for last time slot from the input file.
    aggregateIostatParams(\%hostAgg, \%hostMetrics, $hostName);

    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_DISK_PREFIX, $interval, $samples, \%diskMetrics, \%IOSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics failed for Disk.");
        $retVal = 0;

        goto EXIT;
    }

    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_PREFIX, $interval, $samples, \%hostMetrics, \%IOSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics failed for Host.");
        $retVal = 0;

        goto EXIT;
    }

EXIT:

    return $retVal;
}

# Used to aggregate certain counters at the host level for IOStat command.
#
# @param:
#    $_[0]: [In] Host level aggregated data which we calculated from disks data.
#    $_[1]: [In/Out] Host level final aggregated data.
#    $_[2]: [In] Host name.
#
# @return:
#    Nothing.
sub aggregateIostatParams
{
    my ($hostAggRef, $hostMetricsRef, $hostName) = @_;

    my %hostAgg = %$hostAggRef;

    if (! %hostAgg)
    {
        return;
    }

    if ($hostAgg{"r/s"} ne "")
    {
        # Add read and write totals.
        $hostAgg{l_tps} = $hostAgg{"r/s"} + $hostAgg{"w/s"};
        $hostAgg{l_mbps} = ($hostAgg{"rkB/s"} + $hostAgg{"wkB/s"}) / 1024;
        $hostAgg{l_rmbps} = $hostAgg{"rkB/s"} / 1024;
        $hostAgg{l_wmbps} = $hostAgg{"wkB/s"} / 1024;

        # Delete KB/s data as we have converted in into MB/s data.
        delete($hostAgg{"rkB/s"});
        delete($hostAgg{"wkB/s"});

        # Dump out the aggregated host metrics.
        foreach my $key (keys(%hostAgg))
        {
            $hostMetricsRef->{$key}{$hostName} .= (sprintf("%g", $hostAgg{$key}).",");
        }
    }
}

# Process netstat command output and dump it in physical file.
#
# @param:
#    $_[0]: [In] Host name.
#    $_[1]: [In] Destination folder for Perf data.
#    $_[2]: [In] Destination folder for Aggregate Perf data.
#    $_[3]: [In] Data collection interval.
#    $_[4]: [In] Number of samples.
#    $_[5]: [In] Raw data file name.
#
# @return:
#    0 on success.
#    1 on failure.
sub processLinuxNetstat
{
    my ($hostName, $folder, $aggPerfFolder, $interval, $samples, $fileName) = @_;

    my  $retVal = 0;

    # Used to store information about header in netstat command output.
    my @headerCols = ();

    # Used to store information about data in netstat command output.
    my @dataCols = ();

    # Used to store Network perf metrics. It is used as <counter name, <resource name, performance data>>.
    my %netMetrics = ();

    # Used to store Host perf metrics. It is used as <counter name, <resource name, performance data>>.
    my %hostMetrics = ();

    # It is used to store the data of previous data set and used in delta calculation.
    my %oldValues = ();

    # Used to store index of current data set.
    my $dataSetIndex = 0;

    # If 0 then it means that we are about to read the first line of data part.
    my $first = 0;

    # Used to store host aggregated data.
    my %hostAgg = ();

    if (!open(FILE, $fileName))
    {
        $logObj->error("Unable to open file: [$fileName], error: [$!].");
        $retVal = 0;

        goto EXIT;
    }

    $logObj->debug("Processing file: [$fileName].");

    # Create folder for Host resource.
    unless (defined eval {mkpath("$folder/$HOST_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_PREFIX].");
        $retVal = 1;

        goto EXIT;
    }

    # Create folder for Host Network resource.
    unless (defined eval {mkpath("$folder/$HOST_NET_PREFIX")})
    {
        $logObj->error("Not able to create folder: [$folder/$HOST_NET_PREFIX].");
        $retVal = 1;

        goto EXIT;
    }

    while (<FILE>)
    {
        # Remove the new line character.
        $_ =~ s/\r\n//g;
        $_ =~ s/\n//g;

        # Strip of any leading space
        s/^\s+//;

        # Ignore date line like "Wed Mar  7 07:15:02 UTC 2012"
        if (/\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}/)
        {
            next;
        }

        # There is a ':' after the interface name. Replace ':' with space. Without space between interface name and perf value, data parsing is not possible.
        # Ex: eth0:16764182386 11087660  0  0  0   0      0     0 349213806 5219992  0  0  0   0     0      0
        s/\:/ /g;

        # Ignore partial header lines like "Inter-|   Receive"
        if (/Inter/)
        {
            # Each such line also signifies start of new time
            $dataSetIndex++;
        }

        # Ignore empty lines.
        if (/^$/)
        {
            next;
        }

        # This is header line like
        # face |bytes  packets errs drop fifo frame compressed multicast|bytes  packets errs drop fifo colls carrier compressed
        if (/face/ || /bytes/)
        {
            my $prefix = "Receive";
            my $firstMetric = "";

            # Split on both white space and vertical bar ('|').
            @headerCols = split(/[\s\|]+/, $_);

            # Start from the second header. First one is face
            for (my $i = 1; $i <= $#headerCols; $i++)
            {
                # Each data line contain perf data for both receive data and transmit data with same header for both parts. Data begins with receive
                # data. So we will store first column name for receive data section. Till we get this column name again, data is for receive data.
                # Once we get the same column again, that is the beginning of transmit data section.
                if ($firstMetric eq $headerCols[$i])
                {
                    $prefix = "Transmit";
                }

                # Set name of first metrics.
                if ($i == 1)
                {
                    $firstMetric = $headerCols[$i];
                }

                # Update the column name by prefixing them with data type like receive or transmit.
                $headerCols[$i] = $prefix." ".$headerCols[$i];

                $logObj->debug("Changing [$prefix] [$firstMetric] header column to [".$headerCols[$i]."].");
            }

            # We will assume that the metrics will vary based on Linux kernel. So the number may be different but the names will be fixed.
            $logObj->debug("Number of header columns found: [$#headerCols]");

            $first = 1;
        }
        # Current line is for data.
        else
        {
            # Aggregate host data for previous data set.
            if ($first == 1)
            {
                aggregateNetstatParams(\%hostAgg, \%hostMetrics, $hostName);

                %hostAgg = ();
                $first = 0;
            }

            # Split line for each column.
            @dataCols = split(/\s+/, $_);

            if ($#dataCols < 0)
            {
                next;
            }

            # The number of header and data columns must match otherwise we can't correlate the values.
            if ($#headerCols != $#dataCols)
            {
                $logObj->debug("Data does not match the header columns: [$#headerCols], Data cols [$#dataCols]");

                next;
            }

            # Individual Disk metric
            for (my $i = 1; $i <= $#dataCols; $i++)
            {
                # Convert bytes/s to KB/s
                if ($headerCols[$i] =~ /bytes/)
                {
                    $dataCols[$i] = $dataCols[$i] / 1024;
                }

                # Special treatment for first data set as we need to get final perf data using deltas. So can't generate data for first data set.
                if ($dataSetIndex == 1)
                {
                    # First entry just record silently.
                    $oldValues{$dataCols[0]}{$i} = $dataCols[$i];

                    # Deliberate data hole for the first entry as we cannot collect data for it.
                    $netMetrics{$headerCols[$i]}{$hostName."-".$dataCols[0]} .= ",";
                }
                else
                {
                    # If we get same or more value in current data set then compute delta.
                    if ($dataCols[$i] >= $oldValues{$dataCols[0]}{$i})
                    {
                        # Compute delta and set network metrics with that.
                        $netMetrics{$headerCols[$i]}{$hostName."-".$dataCols[0]} .= sprintf("%g", (($dataCols[$i] - $oldValues{$dataCols[0]}{$i}) / $interval)).",";

                        # Generate host data by aggregating network interface data.
                        if ($headerCols[$i] =~ /packets/ || $headerCols[$i] =~ /bytes/ )
                        {
                            $hostAgg{$headerCols[$i]} += (($dataCols[$i] - $oldValues{$dataCols[0]}{$i}) / $interval);
                        }
                    }
                    # Otherwise treat it as data hole.
                    else
                    {
                        $netMetrics{$headerCols[$i]}{$hostName."-".$dataCols[0]} .= ",";
                    }

                    # Save current values as oldValues to calculate delta for next data set.
                    $oldValues{$dataCols[0]}{$i} = $dataCols[$i];
                }
            }

            # Add network interfaces as resource.
            $resources{$HOST_NET_PREFIX."#".$hostName."-".$dataCols[0]}{"name"} = $hostName."-".$dataCols[0];
            $resources{$HOST_NET_PREFIX."#".$hostName."-".$dataCols[0]}{"relations"} = $HOST_PREFIX."#".$hostName;
        }
    }

    # Aggregate host data.
    aggregateNetstatParams(\%hostAgg, \%hostMetrics, $hostName);

    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_NET_PREFIX, $interval, $samples, \%netMetrics, \%NETSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics failed for Network.");
        $retVal = 1;

        goto EXIT;
    }

    if (!dumpMetrics($folder, $aggPerfFolder, $HOST_PREFIX, $interval, $samples, \%hostMetrics, \%NETSTAT_METRIC_MAP))
    {
        $logObj->error("Call to dumpMetrics failed for Host.");
        $retVal = 1;

        goto EXIT;
    }

EXIT:

    return $retVal;
}

# Used to aggregate certain network counters at the host level.
#
# @param:
#    $_[0]: [In] Host data.
#    $_[1]: [In/Out] Aggregated host data.
#    $_[2]: [In] Host name.
#
# @return:
#    Nothing.
sub aggregateNetstatParams
{
    my ($hostAggRef, $hostMetricsRef, $hostName) = @_;

    my %hostAgg = %$hostAggRef;

    if (!%hostAgg)
    {
        return;
    }

    if ($hostAgg{"Receive bytes"} ne "")
    {
        # Add read and write totals.
        $hostAgg{l_xmbps} = ($hostAgg{"Receive bytes"} + $hostAgg{"Transmit bytes"}) / 1024;
        $hostAgg{l_pckps} = $hostAgg{"Receive packets"} + $hostAgg{"Transmit packets"};

        $hostAgg{l_txmbps} = $hostAgg{"Transmit bytes"} / 1024;
        $hostAgg{l_rxmbps} = $hostAgg{"Receive bytes"} / 1024;

        # Remove counters which are not needed now.
        delete($hostAgg{"Transmit bytes"});
        delete($hostAgg{"Receive bytes"});

        # Dump out the aggregated host metrics
        foreach my $key (keys(%hostAgg))
        {
            if ($hostMetricsRef->{$key}{$hostName} >= 0)
            {
                $hostMetricsRef->{$key}{$hostName} .= (sprintf("%g", $hostAgg{$key}).",");
            }
        }
    }
}

# Dump the processed output to file in MARS format.
#
# @param:
#    $_[0]: [In] Destination folder for perf data.
#    $_[1]: [In] Destination folder for aggregated perf data.
#    $_[2]: [In] MARS prefix for the resource.
#    $_[3]: [In] Data collection interval.
#    $_[4]: [In] Number of samples.
#    $_[5]: [In] Performance data MAP.
#    $_[6]: [In] Map for translating column name in MARS perf counter name.
#
# @return:
#    0 on success.
#    1 on failure.
sub dumpMetrics
{
    my ($folder, $aggFolder, $prefix, $interval, $samples, $metricsRef, $metricMapRef) = @_;

    my %metrics =  %$metricsRef;

    my %metricMap = %$metricMapRef;

    my $retVal = 0;

    for my $systemMetric (keys(%metrics))
    {
        # Look up in metric mapping and get our metric name.
        my $ourMetric = $metricMap{$systemMetric};

        if (!defined($ourMetric) || $ourMetric eq "")
        {
            $logObj->warn("Unknown metric: [$systemMetric].");
            next;
        }

        # Create folder for resource type.
        unless (defined eval {mkpath("$folder/$prefix")})
        {
            $logObj->error("Not able to create folder: [$folder/$prefix].");
            $retVal = 1;

            goto EXIT;
        }

        # Create file for performance counter inside resource folder.
        my $metricFile = "$folder/$prefix/$ourMetric.txt";

        # If file doesn't exist then we have to create the file and put meta-data in it.
        if (! (-e $metricFile))
        {
            if (!open(PFILE_HANDLE, "> $metricFile"))
            {
                $logObj->error("Unable to open [$metricFile] for writing. Error: [$!].");
                $retVal = 1;

                goto EXIT;
            }

            print(PFILE_HANDLE "%meta\n");
            print(PFILE_HANDLE "probe_id:$siteName" . "_" . "$probeName\n");
            print(PFILE_HANDLE "probe_type:Linux\n");
            print(PFILE_HANDLE "probe_site:$siteName\n");
            print(PFILE_HANDLE "probe_server:MARS\n");
            print(PFILE_HANDLE "probe_version:10\n");
            print(PFILE_HANDLE "collection_interval:$interval\n");
            print(PFILE_HANDLE "number_of_samples:$samples\n");
            print(PFILE_HANDLE "%meta\n");
        }
        # Open the file in append mode.
        else
        {
            open(PFILE_HANDLE, ">> $metricFile");
        }

        for my $resource (keys %{$metrics{$systemMetric}})
        {

            $logObj->debug("$prefix#$resource, " . $metrics{$systemMetric}{$resource});


            print(PFILE_HANDLE $prefix."#"."$resource,".$metrics{$systemMetric}{$resource}."\n");

            # If we have to aggregate minute data.
            if ($aggrMinuteData != 1)
            {
                next;
            }

            # Following logic is for aggregating data at minute level.
            my @values = split(',', $metrics{$systemMetric}{$resource});

            # Get the date-time part from the complete path.
            my @tempFolderPath = split(/\//, $folder);
            my $yymmddhhmmss = $tempFolderPath[-1];

            # Get date and time part.
            @tempFolderPath = split(/_/, $yymmddhhmmss);
            my $yymmdd = $tempFolderPath[0];
            my $hhmmss = $tempFolderPath[1];

            # Get second information.
            my $ss = substr($hhmmss, 4, 2);

            my $numSamplesInArray = $#values + 1;

            # Find the number of data points for the current minute which should be used to find the minute level aggregate.
            my $numSamplesInThisInterval = (60 - $ss) / $interval;

            # Create the folder path. Set seconds to "00".
            my $folderPath = (substr($aggFolder, 0, -15)).$yymmdd."_".(substr($hhmmss, 0, 4))."00";

            my $key = "$folderPath#$prefix#$resource#$ourMetric";

            my @subKeys = split(/#/, $key);

            # It get entire path till time-stamp sub-folder.
            my $currentKeyDateTime = $subKeys[0];

            # It existing aggregated data, check whether same key (except time-stamp) already exist with a older time-stamp.
            foreach my $oldKey (keys(%aggregateResource))
            {
                @subKeys = split(/#/, $oldKey);

                # Match all sub-parts of key except time-stamp. If they are not matching then ignore this key.
                if ($prefix ne $subKeys[1])
                {
                    next;
                }

                if ($resource ne $subKeys[2])
                {
                    next;
                }

                if ($ourMetric ne $subKeys[3])
                {
                    next;
                }

                my $oldKeyDateTime = $subKeys[0];

                # Check whether exiting time-stamp is less than current time-stamp. If yes then mark data corresponding to old key for write.
                if ($oldKeyDateTime lt $currentKeyDateTime)
                {
                    $aggregateResource{$oldKey}{towrite} = 1;
                }
            }

            # Keep track of next index in values list.
            my $dataIndex = 0;

            $aggregateResource{$key}{"towrite"} = 0;
            $aggregateResource{$key}{"value"} = 0;
            $aggregateResource{$key}{"numsamples"} = 0;

            # Aggregate all available data for each minute and dump it in physical file.
            while (0 < $numSamplesInArray)
            {
                if (0 >= $numSamplesInThisInterval)
                {
                    # As we have aggregated data for current minute, mark it for writing.
                    $aggregateResource{$key}{"towrite"} = 1;

                    # Set the time-stamp for next minute aggregation and also set the key for that.
                    # Third parameter is aggregation interval.
                    my $yymmdd_hhmmss = adjustDataBoundary($yymmdd, $hhmmss, 60, 1);
                    my $newfolderPath = (substr($folderPath, 0, -15)).$yymmdd_hhmmss;

                    $key = "$newfolderPath#$prefix#$resource#$ourMetric";

                    # For the next aggregation, find out the number of samples needed for next minute.
                    @tempFolderPath = split(/_/, $yymmdd_hhmmss);
                    $yymmdd = $tempFolderPath[0];
                    $hhmmss = $tempFolderPath[1];

                    # Get second information.
                    $ss = substr($hhmmss, 4, 2);

                    $numSamplesInThisInterval = (60 - $ss) / $interval;

                    # Initialize next key.
                    $aggregateResource{$key}{"towrite"} = 0;
                    $aggregateResource{$key}{"value"} = 0;
                    $aggregateResource{$key}{"numsamples"} = 0;
                }

                # Use the next data point for the aggregate operation.
                $aggregateResource{$key}{"value"} = $aggregateResource{$key}{"value"} + $values[$dataIndex];
                $aggregateResource{$key}{"numsamples"} = $aggregateResource{$key}{"numsamples"} + 1;

                # Update different indexes.
                $numSamplesInThisInterval--;
                $numSamplesInArray--;
                $dataIndex++;
            }
        }
    }

    # Dump the aggregated data in physical file.
    aggrMetric(\%aggregateResource);

    close(PFILE_HANDLE);

    return 1;
}

# Find the average value for aggregated data and dump it in file using MARS format.
#
# @param:
#    $_[0]: [In] Map which contain aggregated data information.
#
# @return:
#    Nothing.
sub aggrMetric
{
    my ($aggregatedDataRef) = @_;

    my %aggregatedData = %$aggregatedDataRef;

    my %aggrMetricsHash = ();

    # Used to store the keys which we can delete.
    my @keysToDelete = ();

    foreach (keys %aggregatedData)
    {
        # If data is ready for writing and number of samples are more than 0 then write the data.
        if (exists $aggregatedData{$_}{towrite} && ($aggregatedData{$_}{towrite} == 1) && ($aggregatedData{$_}{numsamples} != 0))
        {
            push(@keysToDelete, $_);

            my ($folder, $prefix, $resource, $metricName) = split(/#/, $_);

            # Find the average value for aggregated data.
            my $average = $aggregatedData{$_}{value} / $aggregatedData{$_}{numsamples};

            $aggrMetricsHash{"$folder"."/"."$prefix"."#$resource"}{$metricName} .= ",". sprintf("%.2f", $average);
        }
    }

    dumpAggrMetricData(60, \%aggrMetricsHash);

    # Delete the key which we have written to physical file.
    foreach my $keyToDelete (@keysToDelete)
    {
        delete($aggregatedDataRef->{$keyToDelete});
    }
}

# Write the processed performance data to file in MARS format.
#
# @param:
#    $_[0]: [In] Data collection interval.
#    $_[1]: [In] Map which contain perf data and related information.
#
# @return:
#    Nothing.
sub dumpAggrMetricData
{
    my ($interval, $aggHashRef) = @_;

    my $retVal = 0;

    my %aggrMetricsHash = %$aggHashRef;

    # Loop for every resource.
    for my $resourceInfo (keys(%aggrMetricsHash))
    {
        # Loop for every metric for a resource.
        for my $metricName (keys%{$aggrMetricsHash{$resourceInfo}})
        {
            # Get details about the resource.
            my @resourceInfoList = split(/#/, $resourceInfo);

            # Create folder for dumping the Perf data.
            unless (defined eval {mkpath($resourceInfoList[0])})
            {
                $logObj->error("Not able to create folder: [$resourceInfoList[0]].");
                $retVal = 1;

                goto EXIT;
            }

            # Set MARS Perf file name.
            my $metricFile = "$resourceInfoList[0]"."/"."$metricName"  .".txt";

            # If file not exist then create a new file and dump meta-data in that.
            if (!(-e $metricFile))
            {
                if (!open(AFILE_HANLDE, "> $metricFile"))
                {
                    $logObj->error("Unable to open [$metricFile] for writing. Error: [$!].");
                    $retVal = 1;

                    goto EXIT;
                }

                # Get number of samples for the data. For aggregate, it would always be 1.
                my $numSamples = ($aggrMetricsHash{$resourceInfo}{$metricName} =~ tr/,//);

                print(AFILE_HANLDE "%meta\n");
                print(AFILE_HANLDE "probe_id:$siteName" . "_" . "$probeName\n");
                print(AFILE_HANLDE "probe_type:Linux\n");
                print(AFILE_HANLDE "probe_site:$siteName\n");
                print(AFILE_HANLDE "probe_server:MARS\n");
                print(AFILE_HANLDE "probe_version:10\n");
                print(AFILE_HANLDE "collection_interval:$interval\n");
                print(AFILE_HANLDE "number_of_samples:$numSamples\n");
                print(AFILE_HANLDE "%meta\n");
            }
            # If file already exist then open it in append mode and no need to write the meta-data again.
            else
            {
                open(AFILE_HANLDE, ">> $metricFile") or die "Unable to open file $metricFile for $!";
            }

            # Get resource type.
            my @prefix = split(/\//, $resourceInfoList[0]);

            # Dump perf data as: <resource-type>#<resource-name>,<perf data>
            print(AFILE_HANLDE "$prefix[-1]"."#"."$resourceInfoList[1]"."$aggrMetricsHash{$resourceInfo}{$metricName}"."\n");
        }
    }

    close(AFILE_HANLDE);
}

# This function is used to return the correct data start boundary for this given data start time. That case come if remote machine send data
# with not exact data start boundary.
#
# @param:
#    $_[0]: [In] Year-Month-Day information for current data.
#    $_[1]: [In] Hour-Minute-Second information for current data.
#    $_[2]: [In] Data collection interval.
#    $_[3]: [In] If 1 then go to next minute boundary.
#
# @return:
#    Return adjusted data start time.
sub adjustDataBoundary
{
    my ($ymd, $hms, $interval, $bForceNxt) = @_;

    # If time interval is 1 then we don't have to adjust the data start time.
    if ($interval == 1)
    {
        return "$ymd"."_"."$hms";
    }

    my $hourVal = substr($hms, 0, 2);
    my $minVal = substr($hms, 2, 2);
    my $secVal = substr($hms, 4, 2);

    my $yearVal = substr($ymd, 0, 4);
    my $dayVal = substr($ymd, 4, 2);
    my $monthVal = substr($ymd, 6, 2);

    # Calculate the time in seconds by which we are misaligned.
    my $timeAlignFactor = $secVal % $interval;

    # If we need to set data start time to next minute or misalignment is such that we are near to next data interval boundary then set the time
    # accordingly.
    if (($bForceNxt) || ($timeAlignFactor > ($interval / 2)))
    {
        # We need to forcibly go to the next minute.
        if ($bForceNxt)
        {
            $secVal = 59;
        }

        # We will shift to next minute.
        if ($secVal eq 59)
        {
            $secVal = "00";

            # We will shift to the next hour.
            if ($minVal eq 59)
            {
                $minVal = "00";

                # We have to move to the next day.
                if ($hourVal eq 23)
                {
                    # Here minute and second have already been zeroed.
                    $hourVal = "00";

                    # Use Perl module to shift to the next day.
                    my $dtDate = Date::Gregorian->new->set_ymd($yearVal, $monthVal, $dayVal);

                    $dtDate->add_days(1);
                    (my $year, my $month, my $day) = $dtDate->get_ymd;

                    $ymd = sprintf("%04d%02d%02d", $year, $month, $day);
                }
                else
                {
                    $hourVal++;
                }
            }
            else
            {
                $minVal++;
            }
        }
        # TODO: We should handle following logic when we add support for interval other than 1 and 60 seconds.
        else
        {
            # Move second value to next data interval boundary.
            $secVal = $secVal + $interval - $timeAlignFactor;
        }
    }
    # We are in the near to previous interval.
    else
    {
        if ($timeAlignFactor ne 0)
        {
            # No need to do complex computation here as we will always use same date time till minute level and not change it.
            $secVal = $secVal - $timeAlignFactor;
        }

        $secVal = sprintf("%02d", $secVal);
    }

    $hms = sprintf("%02d%02d%02d", $hourVal, $minVal, $secVal);

    return "$ymd"."_"."$hms";
}

# Check if we need to generate configuration file.
#
# @param:
#    $_[0]: [In] Host name.
#    $_[1]: [In] Data collection interval between 2 sets of configuration data.
#
# @return:
#    1 if need to generate configuration file. Otherwise 0.
sub isGenerateConfig
{
    my ($chkHostName, $chkInterval) = @_;

    # Indicates whether input host entry is present in configuration file or not.
    my $hostFound = 0;

    # If genConfig = 1 -> Generate configuration file. If genConfig = 0 -> Don't generate.
    my $genConfig = 0;

    # Contain text which needs to be updated in host list file.
    my $updateText = "";

    # If Host file exist then read entries to see whether input host entry is there or not.
    if (-e $HOST_LIST_FILE)
    {
        open(HOSTFILE, "+< $HOST_LIST_FILE") or die $logObj->error("Not able to open file: [$HOST_LIST_FILE].");

        while (<HOSTFILE>)
        {
            # Remove white-space character.
            $_ =~ s/\r\n//g;
            $_ =~ s/\n//g;

            # Remove comments.
            $_ =~ s/#.*//;

            # Remove leading and trailing white-space.
            $_ =~ s/^\s+//;
            $_ =~ s/\s+$//;

            # If length of current line is zero then ignore current line.
            if (0 == length($_))
            {
                next;
            }

            # If found input host name in current line.
            if (/$chkHostName/)
            {
                $hostFound = 1;

                # Get time information for this host from the configuration file.
                my ($fileHost, $fileTmStmp) = split(/,/, $_);
                my ($fileY, $fileM, $fileD, $fileHour, $fileMin, $fileSec) = split(/-/, $fileTmStmp);

                # Find the time difference between time information for host and current time. If it is more than configuration data collection time
                # then we need to update configuration file with new time information.
                if (getDeltaSecs($fileSec, $fileMin, $fileHour, $fileY, $fileM, $fileD) >= $chkInterval)
                {
                    # Update time information for this host.
                    $updateText = $chkHostName .",".getCurrentTime();

                    if (executeCommand("sed -i -e \"s/$_/$updateText/g\" $HOST_LIST_FILE") != 0)
                    {
                        $logObj->error("Replacement of text [$_] with [$updateText] failed for host [$chkHostName].");
                    }
                    else
                    {
                        $logObj->debug("Replacement of text [$_] with [$updateText] succeded for host [$chkHostName].");

                        # As time got changed, need to generate the new configuration file.
                        $genConfig = 1;
                    }
                }

                last;
            }
            else
            {
                $hostFound = 0;
            }
        }
    }
    else
    {
        # Host configuration file not exist so create one and add host entry.
        # Make minute and seconds to zero so that next time configuration file will created at hour boundary.
        open(HOSTFILE, "> $HOST_LIST_FILE") or die $logObj->info("Unable to open file $HOST_LIST_FILE. Error: [$!].");

        print(HOSTFILE $chkHostName.",".getCurrentTime());

        $hostFound = 1;
        $genConfig = 1;
    }

    close(HOSTFILE);

    # At this point, we have Host file exist. If host entry not found then append its entry.
    if ($hostFound == 0)
    {
        open(HOSTFILE, ">> $HOST_LIST_FILE") or die $logObj->error("Unable to open file $HOST_LIST_FILE, error: [$!].");

        print(HOSTFILE "\n");

        print(HOSTFILE $chkHostName.",".getCurrentTime());

        # New host found so generate configuration file.
        $genConfig = 1;

        close(HOSTFILE);
    }

    return $genConfig;
}

# This function is used to check whether raw data processing is already running for Linux probe for the input Linux property file.
#
# @param:
#    None.
#
# @return:
#   0 if no other process is doing raw data processing for same remote host.
#   1 if some other process is also doing raw data processing for same remote host.
sub checkOtherInstanceRunning
{
    # This variable is used to count instances of process_linux script running for input properties file name.
    my $correctProcessCount = 0;

    # This variable is used as the temporary file handle to traverse the output of "ps -df" command.
    my $RESULT_HANDLE;

    $logObj->info("Getting list of processRawData.pl processes running on the system.");

    # Find the list of all processRawData.pl processes running on the system.
    my $storeCmdList = `ps -df | grep processRawData.pl`;

    # Redirect the "grep" output to a temporary file
    open($RESULT_HANDLE, '<', \$storeCmdList);

    # Traverse the "grep" command output using the temporary file handle.
    while (defined (my $resultLine = <$RESULT_HANDLE>))
    {
        # Remove the new line character.
        chomp($resultLine);

        $logObj->info("Process info = [$resultLine].");

        # Check whether this script is running for the input property file.
        if ($resultLine =~ /^(.+)$propFileName(.*)(\s*)$/)
        {
            $correctProcessCount++;
        }
    }

    close($RESULT_HANDLE);

    if (1 == $correctProcessCount)
    {
        $logObj->info("No other instance of process_linux found which is running for input properties file.");
        return 0;
    }
    else
    {
        $logObj->warn("More than one instance of process_linux found which is running for input properties file.");
        return 1;
    }
}

# This function is used to dump aggregated resource data at the end.
#
# @param:
#    None.
#
# @return:
#   0 if Success
#   1 if Error
sub dumpAggrResources
{
    # Dump aggregated resource data in the hash file. So that next invocation of process_linux will use it to continue data aggregation.
    # TODO: What if we are not able to open this file. Return 1 in that case.
    if (open(FH, '>', $HASH_FILE) == 0)
    {
        $logObj->error("Failed to open file: [$HASH_FILE]");
        return 1;
    }

    print(FH Dumper(\%aggregateResource));

    $logObj->debug("Content of aggregateResource ".Dumper(\%aggregateResource));

    # Close hash file handle.
    close(FH);

    return 0;
}

# This function is used to read the input .properties file.
#
# @affected global variables :
#   $probeId
#   $siteName
#   $probeName;
#
# @return:
#   0 if Success
#   1 if Error
sub readPropertyFile
{
    # This variable is used to store the return code for this function.
    my $retVal = 0;

    $logObj->info("Reading file: [$propFileName].");

    open(PROP_FILE, $propFileName) or die $logObj->error("Not able to open the file: [$propFileName].");

    # Loop through the content of .properties file.
    while (<PROP_FILE>)
    {
        # Remove the new line character.
        $_ =~ s/\r\n//g;
        $_ =~ s/\n//g;

        # Look for Probe name.
        if ($_ =~ /^ProbeName=(\S+)(\s*)$/)
        {
            $probeName = $1;
        }
        # Look for Site name.
        elsif ($_ =~ /^SiteName=(.+)(\s*)$/)
        {
            $siteName = $1;
        }
        else
        {
            # Current line is of no use. So ignore it.
        }
    }

    if ("" eq $probeName)
    {
        $logObj->error("Probe name is not present in the properties file.");
        $retVal = 1;

        goto EXIT;
    }

    if ("" eq $siteName)
    {
        $logObj->error("Site name is not present in the properties file.");
        $retVal = 1;

        goto EXIT;
    }

    $probeId = "Linux_". $probeName;

EXIT:

    close(PROP_FILE);

    return $retVal;
}

# TODO: Check whether we can do this using Date::Calc.
# This function is used to find the time difference in seconds for input time and current system time.
#
# @param:
#    $_[0]: [In] Second part of time.
#    $_[1]: [In] Minute part of time.
#    $_[2]: [In] Hour part of time.
#    $_[3]: [In] Year part of time.
#    $_[4]: [In] Month part of time.
#    $_[5]: [In] Day part of time.
#
# @return:
#    Time difference in seconds.
sub getDeltaSecs
{
    my ($fileSec, $fileMin, $fileHour, $fileY, $fileM, $fileD) = @_;
    my ($nowSec, $nowMin, $nowHour, $nowD, $nowM, $nowY, $wDay, $yDay, $isDst) = localtime(time);

    $logObj->info("Input time: [$fileSec, $fileMin, $fileHour, $fileY, $fileM, $fileD].");
    $logObj->info("Current time: [$nowSec, $nowMin, $nowHour, $nowD , $nowM, $nowY].");

    use Time::Local qw /timelocal/;

    # TODO: Why we are doing "+ 0".
    my $epochFile = timelocal($fileSec, $fileMin, $fileHour, $fileD + 0, $fileM - 1, $fileY);
    my $epochNow = timelocal($nowSec, $nowMin, $nowHour, $nowD + 0, $nowM, $nowY + 1900);

    $logObj->info("epochfile : [$epochFile], epochnow : [$epochNow].");

    my $diffTime = abs($epochNow - $epochFile);
    $logObj->info("diffTime :[$diffTime]");

    return $diffTime;
}

# Time string in the format YYYY-MM-DD-HH-MM-SS where MM and SS set to 00.
#
# @param:
#    None.
#
# @return:
#    Time is specific format.
sub getCurrentTime
{
    my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);

    return sprintf("%04d-%02d-%02d-%02d-%02d-%02d", $year + 1900, $mon + 1, $mday, $hour, 0, 0);
}

# Used to run a system command.
#
# @param:
#   $_[0] - Command to execute.
#
# @return:
#   0 if Success
#   1 if Error
sub executeCommand
{
    # Read input parameter.
    my ($command) = @_;

    # This variable is used to store the return code for this function.
    my $retVal = 0;

    system($command);

    if (($? == -1) || ($? & 127) > 0)
    {
        $logObj->error("Failed to execute command [$command]. Error code: [".($? & 127)."].");
        $retVal = 1;

        goto EXIT;
    }

EXIT:

    return $retVal;
}