# Copyright 2008-2012 Cumulus Systems Incorporated. # All Rights Reserved. # This file contains the code to process the Eternus Raw Performance data and convert it into the MARS Probe data format. # # @param : IP Address - Eternus Storage IP Address # # @return : In case of Success, returns 0. In case of Failure, returns 1. # # Sample command: perl eternusPerfDataProcessor.pl #!/usr/bin/perl -w # It is used for strict compilation. use strict; # It is used for specifying the location of common module. use lib "/usr/local/megha/lib/eternus"; # It is used for using the functions of common module use commonModule; # It has a function which is used for incrementing date by one day. use Date::Gregorian; # It is used to get base file name from the complete file name. use File::Basename; # Used to get current time. use Time::HiRes q/gettimeofday/; use POSIX "strftime"; # Used for incrementing a given timestamp by 15 minutes. use Time::Local qw(timelocal); # It is used to dump the complex data structure. use Data::Dumper; # It is used for rmtree function. use File::Path; # It is used to create a folder recursively. use File::Path qw(mkpath); # It is used to copy the file from one location to other. use File::Copy; # ---------------------------------------------------------------------------------------------------------------------------------------------------- # GLOBAL Variables # ---------------------------------------------------------------------------------------------------------------------------------------------------- # This variable is used to store the return status of function calls. my $status = 0; # This variable is used to store the base path where Eternus Raw Perf data is present in different folders based on the Store command start timestamp. # We use "stxpmonitor" command to extract a Resource specific data from this Raw data. my $rawPerfDataBasePath = ""; # This variable is used to store the start time for the Performance data which is stored in the current MARS counter performance data file. my $perfStartTime = 0; # This variable is used to store the Probe Site name. my $marsSiteName = ""; # This variable is used to store the Eternus Storage IP Address. my $eternusIpAddress = ""; # This variable is used to store the current MARS Probe Version number. my $probeVersion = "6.0"; # This variable is used to store the complete name of Eternus Performance utility. my $eternusPerfUtil = "/opt/FJSVstxp/bin/stxpmonitor"; # This array is used to store the name of all Eternus Resource types for which we are going to collect the Performance data. my @resourceTypeList = (); # This hash is used to store any special part of "stxpmonitor" command that needs to be run for the corresponding Resource. my %resourceCmd = (); # This hash is used to store the start regular expression for each Eternus resource. This regular expression is used to find the header of Performance # data in raw performance data in Eternus format. my %resourceStartRegExHash = (); # This hash is used to store the performace regular expression for each Eternus resource. This regular expression is used to find the Performance data # in raw performance data in Eternus format. my %resourcePerfRegExHash = (); # This hash is used to store the Perl expression which is used to generate the Eternus resource instance name at run time. my %resourceInstNameHash = (); # This hash is used to store the Perl expression which is used to store the Eternus resource name. my %resourceNameHash = (); # This hash is used to store the Perl expression which is used to find the correct counter value from the raw performance data at run time. my %resourcePerfCounterHash = (); # This hash is used to store the complete Performance data to generate all MARS specific files/folders which contains the Performance data for 15 # minutes. my %perfData = (); # This hash is used to store the list of Eternus files which are used to generate the Performace data for current 15 Perf data points. Here key is the # pmd file name and value is corresponding Perf folder name. my %usedFileNameList = (); # This variable is used as the FILE handle for current counter performance data file. This file contains counter specific performance data for 15 # minutes. my $PERF_FILE; # This variable is used to check whether there is raw Eternus performance data available for at least 15 minutes. my $rawPerfDataAvailable = 0; # This variable is used to store the complete name of file which contain the Eternus resource specific data used to collect the MARS performance data. my $perfMetadataFile = ""; # This variable is used to store the complete name of file where Eternus raw performance data will be dumped. my $rawPerfDumpFile = ""; # This variable is used to store full name of Raw perf folder which is selected to be used for MARS performance data collection for the current cycle. my $selectedFolderForPerfData = ""; # This variable is used to store number of pmd files present in the $selectedFolderForPerfData folder. my $pmdFilesForPerfData = 0; # This variable is used to store the path of folder which is used to temporary store the 15 Raw Perf data files so that we can generate the MARS Perf # data for the oldest 15 Raw Perf data file available. my $tempRawPerfDataPath = ""; # This list is used to store the complete name of pmd and dcd files which are used to generate the MARS Perf data for the current cycle. These files # need to be moved to a temporary folder so that we can process the oldest files first. my @rawPerfFileForMove = (); # This list is used to store the complete name of pmd and dcd files which are used to generate the MARS Perf data for the current cycle. These files # need to be copied to a temporary folder unlike files in @rawPerfFileForMove because these files will be required in every MARS Perf data collection # cycle. my @rawPerfFileForCopy = (); # This variable is used to store the name of Eternus Storage. It is used to generate the MARS Resource name correctly. my $eternusStorageName = ""; # This variable is used to store the complete name of file where conf data from Eternus Storage will be dumped. my $confDumpFile = ""; # This variable is used to store the location of Eternus runtime files. my $eternusRunTimeLocation = ""; # This variable is used to store the complete path of MARS app.properties file. my $appPropertiesFilePath = ""; # This variable is used to store the path of MARS properties files. It is required to read the the Eternus Storage credentials from the .properties # file specific to the input Eternus IP address. my $propFilePath = ""; # This variable is used to store the complete name of .properties file which corresponds to the input Eternus IP address. my $propFileName = ""; # This variable is used to store the path of Probe folder in db/probe. my $eternusProbeFolder = ""; # This variable is used to store the Eternus Storage User-Id which is used to get the Storage Configuration data. my $eternusConfUserId = ""; # This variable is used to store the Eternus Storage Password corresponding to $eternusConfUserId. my $eternusConfPassword = ""; # This variable is used to store the timestamp of first selected pmd file till day part with format as yyyymmdd. It is needed to convert the perf data # timestamp from hhmmss to yyyymmddhhmmss format as we are not getting yyyyddmm information while parsing the transformed raw perf data. my $timeTrace = ""; # It is used as the backup of $timeTrace value for one MARS perf data collection cycle. Reason is that while processing the perf data for a resource, value # of $timeTrace may get change if perf data timestamp crosses hour boundary. So before start processing next resource perf data, we need to reset the # value of $timeTrace with the original value set for that MARS perf data collection cycle. my $timeTraceCopy = ""; # It is used to make entry of those files which are missing within a range of 15 minutes. my $finalTimeStamp = ""; # This is used to store the minute part of perf data timestamp which is current get processed. It is used to check whether next perf data timestamp # crossed the hour boundary. my $previousTimeTraceMin = ""; # This is used to store the hour part of perf data timestamp which is current get processed. It is used to check whether next perf data timestamp # crossed the hour boundary. my $previousTimeTraceHour = ""; # This variable is used to store the maximum number of pmd files which are required for one iteration of MARS perf data generation. my $PMD_FILES_FOR_ITERATION = 16; # This variable is used to store the Eternus Storage model from the properties file. my $storageModel = ""; # This variable is used to store the complete name of log file. my $logFile = ""; # This variable is used to store the file handle of log file. my $LOG_HANDLE; # This variable is used to store the base folder path on the MARS probe. my $baseFolder = ""; # This variable holds the path from where configuration data can be collected for devices supporting only MICC. my $miccUtil = "/opt/FJSVstxs/bin/stxsconfiguration"; # This is the maximum Perf value which we are treating as valid value. Anything beyond that it is not a valid Perf value. my $MAX_VALID_PERF_VALUE = "999999"; # ---------------------------------------------------------------------------------------------------------------------------------------------------- # Main function # ---------------------------------------------------------------------------------------------------------------------------------------------------- # Initialize the environment. $status = init(); if (0 != $status) { logMsg("Error: Call to init() failed."); goto EXIT; } # Read each Resource specific information which is used to generate the Eternus Performance data. $status = readPerfMetadataFile(); if (0 != $status) { logMsg("Error: Call to readPerfMetadataFile() failed."); goto EXIT; } # Keep on generating Performance data in MARS format unless no raw performance data left. while (1) { # Check whether there is enough Eternus raw performance data present for at least 15 minutes. $status = checkRawPerfDataStatus($rawPerfDataAvailable); if (0 != $status) { logMsg("Error: Call to checkRawPerfDataStatus() failed."); goto EXIT; } # If no more raw performance data present then exit. if (0 == $rawPerfDataAvailable) { logMsg("Info: No more raw performance data available. Exiting."); goto EXIT; } # Collect the Performance data for last 15 minutes. $status = collectMarsPerfData(); if (0 != $status) { logMsg("Error: Call to collectMarsPerfData() failed."); goto EXIT; } # Generate the Performance data in MARS format for last 15 minutes. $status = dumpPerfData(); if (0 != $status) { logMsg("Error: Call to dumpPerfData() failed."); goto EXIT; } # Delete the raw Perf data files for which we generated the Perf files in MARS format. $status = deleteRawPerfData(); if (0 != $status) { logMsg("Error: Call to deleteRawPerfData() failed."); goto EXIT; } } EXIT: # De-init the environment. deInit(); exit $status; # ---------------------------------------------------------------------------------------------------------------------------------------------------- # Sub-routines # ---------------------------------------------------------------------------------------------------------------------------------------------------- # This function is used to initialize the environment to run the Perl script correctly. # # @affected global variables : # $eternusIpAddress # $perfMetadataFile # $rawPerfDumpFile # $confDumpFile # $appPropertiesFilePath # $propFilePath # # @return : # 0 if Success # 1 if Error sub init { # This variable is used to store the return value of function calls. my $retVal = 0; # Set MARS base folder. $baseFolder = "/usr/local/megha"; # If number of command line arguments are not correct then quit. if (1 != ($#ARGV + 1)) { logMsg("Error: Number of arguments to eternusPerfDataProcessor.pl are not correct. Total input arguments = ".($#ARGV + 1)); $retVal = 1; goto EXIT; } # Get the Eternus Storage IP Addrss from command line argument. $eternusIpAddress = $ARGV[0]; logMsg("Info: Input IP Address = [$eternusIpAddress]."); # TODO: Test it after making changes in build script. # Set different paths. $propFilePath = $baseFolder."/conf/probe"; logMsg("Info: Properties file path: [$propFilePath]."); # Get the name of .properties file corresponding to Eternus Storage. $status = getPropertyFileName($propFilePath, $eternusIpAddress, $propFileName); if (0 != $status) { logMsg("Error: Call to getPropertyFileName() failed."); $retVal = 1; goto EXIT; } # Read the .properties file corresponding to Eternus Storage. $status = readPropertyFile(); if (0 != $status) { logMsg("Error: Call to readPropertyFile() failed."); $retVal = 1; goto EXIT; } # Construct the complete file name for Eternus Performance metadata file depending on the Storage model. if ($storageModel =~ /^ET0[89]2.*$/) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX80S2.conf"; } elsif ($storageModel =~ /^ET06[^2].*$/) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX60.conf"; } elsif (($storageModel =~ /^ET442.*$/) || ($storageModel =~ /^ET872.*$/)) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX440S2.conf"; } elsif (($storageModel =~ /^E4.*$/) || ($storageModel =~ /^ET8.[^2].*$/) || ($storageModel =~ /^E8.*$/)) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX4000.conf"; } elsif (($storageModel =~ /^E2.*$/) || ($storageModel =~ /^ET08[^2].*$/)) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX2000.conf"; } elsif ($storageModel =~ /^ET412.*$/) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX410S2.conf"; } elsif ($storageModel =~ /^ET4.[^2].*$/) { $perfMetadataFile = $baseFolder."/lib/eternus/eternusPerfMetadata_DX410.conf"; } else { logMsg("Error: Invalid Storage model found. Model = [$storageModel]."); $retVal = 1; goto EXIT; } logMsg("Info: Eternus Perf metadata file = [$perfMetadataFile]."); # Set the path of Probe folder in db/probe. $eternusProbeFolder = $baseFolder."/db/probe/".$marsSiteName."/Eternus_".$eternusIpAddress; logMsg("Info: Creating folder: [$eternusProbeFolder]."); # Create the folder for this Eternus Probe in the probe folder. unless (defined eval {mkpath($eternusProbeFolder)}) { logMsg("Error: Not able to create folder: [$eternusProbeFolder]."); $retVal = 1; goto EXIT; } # Set the file name where the Eternus Storage data will be dumped. $confDumpFile = $eternusProbeFolder."/eternusVerification.dump"; logMsg("Info: Eternus Conf data dump file: [$confDumpFile]."); # Set the complete path of MARS app.properties file. $appPropertiesFilePath = $baseFolder."/conf/app.properties"; logMsg("Info: app.properties file path: [$appPropertiesFilePath]."); # Read the content of app.properties file. $retVal = readAppPropertyFile($appPropertiesFilePath, $eternusIpAddress, $eternusRunTimeLocation); if (0 != $retVal) { logMsg("Error: Call to readAppPropertyFile() failed."); $retVal = 1; goto EXIT; } # If the Storage Model Name is out of the following then we need to call getEternusStorageNameSsh function. if (($storageModel =~ /^ET0[89]2.*$/) || ($storageModel =~ /^ET06[^2].*$/) || ($storageModel =~ /^ET442.*$/) || ($storageModel =~ /^ET872.*$/) || ($storageModel =~ /^ET08[^2].*$/) || ($storageModel =~ /^ET09[^2].*$/) || ($storageModel =~ /^ET412.*$/) || ($storageModel =~ /^ET062.*$/) || ($storageModel =~ /^ET812.*$/)) { # Get Eternus Storage name. $status = getEternusStorageNameSsh(); if (0 != $status) { logMsg("Error: Call to getEternusStorageNameSsh() failed."); goto EXIT; } } # If the Storage Model Name is out of the following then we need to call getEternusStorageNameMicc function. elsif (($storageModel =~ /^E2.*$/) || ($storageModel =~ /^E4.*$/) || ($storageModel =~ /^ET8.[^2].*$/) || ($storageModel =~ /^E8.*$/) || ($storageModel =~ /^ET4.[^2].*$/)) { # Get Eternus Storage name. $status = getEternusStorageNameMicc(); if (0 != $status) { logMsg("Error: Call to getEternusStorageNameMicc() failed."); goto EXIT; } } else { logMsg("Error: Invalid Storage model found. Model = [$storageModel]."); $retVal = 1; goto EXIT; } # Set the path which is used to create the Raw Perf data folders. $rawPerfDataBasePath = $eternusProbeFolder."/PerfRawData"; logMsg("Info: Eternus Raw Perf data folder = [$rawPerfDataBasePath]."); # Set the path of file which is used to store the transformed data from the raw Perf data file. $rawPerfDumpFile = $rawPerfDataBasePath."/eternusRawPerfData.dump"; logMsg("Info: Eternus Raw Perf data file = [$rawPerfDumpFile]."); # Set the path of the folder which is used to temporary store the Raw Perf data files so that we can generate the MARS Perf data for the oldest # Raw Perf data available. $tempRawPerfDataPath = $rawPerfDataBasePath."/TempRawPerfData"; logMsg("Info: Eternus Temp Raw Perf data folder = [$tempRawPerfDataPath]."); logMsg("Info: Creating Folder: [$tempRawPerfDataPath]."); unless (defined eval {mkpath($tempRawPerfDataPath)}) { logMsg("Error: Not able to create Perf date folder."); $retVal = 1; goto EXIT; } EXIT: return $retVal; } # This function is used to read the .properties file which corresponds to the input Eternus IP address. It reads the Eternus Storage User-Id and # Password from the .properties file which are used to connect to the Eternus Storage. # # @affected global variables : # $eternusConfUserId # $eternusConfPassword # $marsSiteName # $storageModel # # @return : # 0 if Success # 1 if Error sub readPropertyFile { # This variable is used to store the return code for this function. my $retVal = 0; logMsg("Info: Reading file: [$propFileName]."); open(my $PROP_FILE, $propFileName) or die logMsg("Error: Not able to open the file: [$propFileName]."); # Loop through the content of .properties file. while (<$PROP_FILE>) { # Remove the new line character. chomp; # Look for Eternus UserId for getting Configuration data. if ($_ =~ /^ConfUserId=(\S+)(\s*)$/) { $eternusConfUserId = $1; } # Look for Eternus Password for getting Configuration data. elsif ($_ =~ /^ConfPassword=(\S+)(\s*)$/) { $eternusConfPassword = $1; } elsif ($_ =~ /^SiteName=(.+)(\s*)$/) { $marsSiteName = $1; } elsif ($_ =~ /^StorageModel=(\S+)(\s*)$/) { $storageModel = $1; } else { # Current line is of no use. So ignore it. } } if ("" eq $eternusConfUserId) { logMsg("Error: Eternus Configuration User-Id not present in the properties file."); $retVal = 1; goto EXIT; } if ("" eq $eternusConfPassword) { logMsg("Error: Eternus Configuration Password not present in the properties file."); $retVal = 1; goto EXIT; } if ("" eq $marsSiteName) { logMsg("Error: MARS Site name is not present in the properties file."); $retVal = 1; goto EXIT; } EXIT: close($PROP_FILE); return $retVal; } # This function is used to read the file which contains Eternus resource specific information which is used to generate the Eternus Performance data # in MARS format. This information includes how to extract the Eternus performance data from the raw performance data and how to generate the resource # names. # # @affected global variables : # @resourceTypeList # %resourceStartRegExHash # %resourcePerfRegExHash # %resourceInstNameHash # %resourcePerfCounterHash # # @return : # 0 if Success # 1 if incomplete data found in the file. sub readPerfMetadataFile { # This variable is used to store the FILE handle of "eternusPerfMetadata.conf" file. my $RESULT_HANDLE; # This variable is used to store the current sequence number in the Perf metadata file to make sure that correct data is present in the file. my $seqNum = 0; # Open the file. open($RESULT_HANDLE, '<', $perfMetadataFile) or die logMsg("Error: Not able to open the file: [$perfMetadataFile]."); logMsg("Info: Opened file: [$perfMetadataFile]."); # Loop through content of file. while (<$RESULT_HANDLE>) { # Remove the new line character. chomp; logMsg("Info: Perf Metadata file data = [$_]."); # Check for Resource Metadata start line. if ((/^(\s*)ResourceDataStart(\s*)$/) && (0 == $seqNum)) { logMsg("Info: Found Start of Resource data"); $seqNum = 1; } # Check for resource type. elsif ((/^(\s*)(\w+):Type=(\w+)(\s*)$/) && (1 == $seqNum)) { logMsg("Info: Resource type: [$2]."); $seqNum = 2; # Save resource type in the global list. push(@resourceTypeList, $2); } # Check for resource name. elsif ((/^(\s*)(\w+):ResName=(\w+)(\s*)$/) && (2 == $seqNum)) { logMsg("Info: Resource name: [$2]."); $seqNum = 3; # Save resource name in the global list. $resourceNameHash{$2} = $3; } # Check for resource specific command. elsif ((/^(\s*)(\w+):Cmd=(.*)$/) && (3 == $seqNum)) { logMsg("Info: Resource Cmd: [$3]."); $seqNum = 4; # Update global hash to store resource specific command for the resource. $resourceCmd{$2} = $3; } # Check for resource start RegEx. elsif ((/^(\s*)(\w+):StartRegEx=(.+)$/) && (4 == $seqNum)) { logMsg("Info: Resource Start RegEx: [$3]."); $seqNum = 5; # Update global hash to store start RegEx for the resource. $resourceStartRegExHash{$2} = $3; } # Check for resource performance RegEx. elsif ((/^(\s*)(\w+):PerfRegEx=(.+)$/) && (5 == $seqNum)) { logMsg("Info: Resource Perf RegEx: [$3]."); $seqNum = 6; # Update global hash to store Perf RegEx for the resource. $resourcePerfRegExHash{$2} = $3; } # Check for resource instance name info. elsif ((/^(\s*)(\w+):Name=(.+)$/) && (6 == $seqNum)) { logMsg("Info: Resource instance name: [$3]."); $seqNum = 7; # Update global hash to store resource instance name info. $resourceInstNameHash{$2} = $3; } # Check for resource Perf data info. There could be one or more Perf data associated with a Eternus resource. elsif ((/^(\s*)(\w+):Perf=(.+)=(\w+)(\s*)$/) && ((7 == $seqNum) || (8 == $seqNum))) { logMsg("Info: Perf Counter name : [$4]. Counter value = [$3]."); $seqNum = 8; my %tempHash = (); # There could be one or more Performance counters associated with a single Resource. So check if the current resource already has # information for some counter. In that case, update the existing list. my $perfCounterList = (); if (exists $resourcePerfCounterHash{$2}) { $perfCounterList = $resourcePerfCounterHash{$2}; } $perfCounterList->{$4} = $3; # Re-associate the updated counter list with the resource. $resourcePerfCounterHash{$2} = \%$perfCounterList; } # Check for Resource Metadata end line. elsif ((/^(\s*)ResourceDataEnd(\s*)$/) && (8 == $seqNum)) { logMsg("Info: Found End of Resource data"); # Initialize the sequence number to get the information about next resource. $seqNum = 0; } # Empty line. elsif (/^$/) { # Do nothing. } # Comment line elsif (/^#(.+)$/) { # Do nothing. } else { logMsg("Warning: Invalid data found the Perf Metadata file. Data = [$_]."); } } close($RESULT_HANDLE); # If there is incomplete data in the file then the sequence number will not be 0. So return with error. if (0 != $seqNum) { logMsg("Error: Incomplete data found in Perf Metadata file."); return 1; } else { return 0; } } # This function is used to parse the Eternus raw Performance data and store the parsed information in a global data structure. # # @affected global variables : # None # # @return : # 0 if Success # 1 if incomplete data found in the file. sub collectMarsPerfData { # This variable is used to store the return value of function calls. my $retVal = 0; # This variable is used to indicate the beginning of Eternus Performance data in the raw performance data file. my $perfDataStart = 0; # This variable is used to store the number of performance data points find till now. my $iterationFound = 0; # This variable is used in the backgroup to evalute an Perl expression given in the Resource metadata file. my $resName = ""; # This variable is used in the backgroup to evalute an Perl expression given in the Resource metadata file. my $perfVal = ""; logMsg("Info: Starting collection of MARS Perf data."); logMsg("Info: Raw Perf folder: [$selectedFolderForPerfData]. File in folder: [$pmdFilesForPerfData]"); # Loop through each Resource whose performance data needs to be collected. foreach my $resourceType (@resourceTypeList) { logMsg("Info: Resource type = [$resourceType]."); logMsg("Info: Cmd = [$resourceCmd{$resourceType}]."); logMsg("Info: Start Reg Ex = [$resourceStartRegExHash{$resourceType}]."); logMsg("Info: Perf Reg Ex = [$resourcePerfRegExHash{$resourceType}]."); logMsg("Info: Resource name = [$resourceInstNameHash{$resourceType}]."); # Get the performance counter information about the current resource. my $perfInfo = $resourcePerfCounterHash{$resourceType}; # print Dumper(\%resourcePerfCounterHash); # print Dumper(\$perfInfo); # Get the available Eternus raw performance data for the last 15 minutes. my $cmd = $eternusPerfUtil." ".$resourceType." -r ".$tempRawPerfDataPath." -p count=15 ".$resourceCmd{$resourceType}." > ".$rawPerfDumpFile; logMsg("Info: Running command: [$cmd]."); system($cmd); # TODO: Check whether it is required. Does system call return only after execution completed. logMsg("Info: Sleeping for 2 seconds."); sleep(2); open(PERF_DUMP, $rawPerfDumpFile) or die logMsg("Error: Not able to open the file: [$rawPerfDumpFile]."); $iterationFound = 0; $perfDataStart = 0; # Loop through the content of file until we found 15 performance data points. while () { # If we found 15 Perf data points then exit from the loop. if (15 == $iterationFound) { last; } # Remove the new line character. chomp; # Skip this line as actual performance data will start from the next line. if (1 == $perfDataStart) { $perfDataStart = 2; next; } # Look for starting header in the dump file. If we found that line then the actual performance data start at next to next line. if (/$resourceStartRegExHash{$resourceType}/) { logMsg("Info: Found Perf data header."); $perfDataStart = 1; # Skip this line as it doesn't contain any useful data. next; } # Process the current line only if reached at the actual Performance data in the dump file. if (2 == $perfDataStart) { # Look for performance data. if (/$resourcePerfRegExHash{$resourceType}/) { # Save the name of corresponding .pmd file so that we can verify the Perf data time at the end and also delete the .pmd file. $retVal = storePerfFileName($2, $selectedFolderForPerfData); if (0 != $retVal) { logMsg("Error: Call to storePerfFileName() failed."); goto EXIT; } # Found the resource instance name by evaluting the perl expression which is like "$resName="etDisk#CE-Disk#".$4". So here # depending on the value of $4, we can compute the correct resource name. my $resourceName = eval($resourceInstNameHash{$resourceType}); # Get the list of Performance counters associated with this resource. my $perfCounterInfo = $resourcePerfCounterHash{$resourceType}; # Loop through every Performance counter information to collect counter perf data. foreach my $counterName (keys %$perfCounterInfo) { updatePerfData($resourceType, $resourceName, $counterName, $2, eval($perfCounterInfo->{$counterName})); } } # If found empty line. elsif (/^$/) { # Increment the number of data point found till now. $iterationFound = $iterationFound + 1; } else { # Current line is of no use. So ignore it. } } } # While updating the Perf data for a particular resource, $timeTrace value may get change if perf data timestamp crosses the hour boundary. # So we need to again initialize $timeTrace with the timestamp of first selected pmd file so that we can update the correct perf data # timestamp for next resource. $timeTrace = $timeTraceCopy; # Following variables also need to be reset before start processing the perf data for next resource as their value also get changed everytime # while processing one perf data. $previousTimeTraceMin = ""; $previousTimeTraceHour = ""; } logMsg("Info: Completed collection of MARS Perf data."); EXIT: close (PERF_DUMP); return $retVal; } # This function is used to update the global hash variable which contain all information which is required to generate one set of MARS Perf data. # # @param : # $_[0] - Resource type. # $_[1] - Resource instance name. # $_[2] - Counter name. # $_[3] - Time for which we collected the Perf data. Its format is "hh:mm:ss". # $_[4] - Counter perf data collected at $_[3] time. # # @affected global variables : # $timeTrace # # @return : # 0 if Success # 1 if incomplete data found in the file. sub updatePerfData { # Read Input Parameters. my ($resourceType, $resourceName, $counterName, $time, $counterVal) = @_; # Convert input time value from 'hh:mm:ss' format to 'yyyymmddhhmmss' format. The initial value of timeTrace is set from the name of first pmd # file selected for the current iteration. We are doing this conversion so that we can properly sort the perf data points for the current # iteration when data points are crossing the day boundary. my $hour = substr($time, 0, 2); my $min = substr($time, 3, 2); my $sec = substr($time, 6, 2); # In this condition, we need to increment the value of timeTrace by one day so that we get proper timestamp for each resource as the day boundary is crossed. if ((23 == $previousTimeTraceHour) && ($previousTimeTraceMin > $min)) { my $timeTraceYear = substr($timeTrace, 0, 4); my $timeTraceMonth = substr($timeTrace, 4, 2); my $timeTraceDay = substr($timeTrace, 6, 2); # Date::Gregorian Module of perl has a function add_days() through which we can increment day by one. my $dtDate = Date::Gregorian->new->set_ymd($timeTraceYear, $timeTraceMonth , $timeTraceDay); # Adding a day to the given timeTrace. $dtDate->add_days(1); # Extracting value of year, month, and day after incrementing the day by one. (my $newYear, my $newMonth, my $newDay) = $dtDate->get_ymd(); # Updating timeTrace with the new incremented value. We are doing it so that when we again call this function, we will get the new date from # timeTrace (if we are crossing the day boundary). $timeTrace = sprintf("%04d%02d%02d", $newYear, $newMonth, $newDay); } # Add yyyymmdd value in the input timestamp. $time = $timeTrace.$hour.$min.$sec; # If we get '-' as the counter value then ignore that value. if ("-" eq $counterVal) { $counterVal = "0"; } # If we get Performance data value more than 999999 then treat it as invalid value because this huge Perf data could be the result of some malfunction on # Storage side. if ($counterVal gt $MAX_VALID_PERF_VALUE) { $counterVal = ""; } #print Dumper(\%perfData); # Check whether Hash for storing the Perf data for Resource "$resourceType" is already exist or not. If yes then use it. my $resourcePerfHash = (); if (exists $perfData{$resourceType}) { $resourcePerfHash = $perfData{$resourceType}; } #print Dumper(\%$resourcePerfHash); # Check whether Hash for storing the Perf data for Counter "$counterName" in Resource "$resourceType" is already exist or not. If yes then use it. my $counterPerfHash = (); if (exists $resourcePerfHash->{$counterName}) { $counterPerfHash = $resourcePerfHash->{$counterName}; } # Check whether Hash for storing the Perf data for Resource instance "$resourceName" for Counter "$counterName" is already exist or not. If yes # then use it. my $resourceInstancePerfHash = (); if (exists $counterPerfHash->{$resourceName}) { $resourceInstancePerfHash = $counterPerfHash->{$resourceName}; } #print Dumper(\%$resourceInstancePerfHash); #print Dumper(\%$counterPerfHash); # Check whether Perf value for Time "$time" for Counter "$counterName" in Resource instance "$resourceName" is already exist or not. If yes then # use it. my $perfVal = 0; if (exists $resourceInstancePerfHash->{$time}) { $perfVal = $resourceInstancePerfHash->{$time}; } $perfVal = $counterVal; # Now update the final Perf data container. $resourceInstancePerfHash->{$time} = $perfVal; $counterPerfHash->{$resourceName} = \%$resourceInstancePerfHash; $resourcePerfHash->{$counterName} = \%$counterPerfHash; $perfData{$resourceType} = \%$resourcePerfHash; #print Dumper(\%perfData); # Update the value of current minute and hour so that if next perf data timestamp crosses the hour boundary, we can check that. $previousTimeTraceMin = $min; $previousTimeTraceHour = $hour; } # This function is used to delete raw Performance data files for which we have generated MARS Performance data files. So now these files are not # required. # # @affected global variables : # %usedFileNameList # %perfData # @rawPerfFileForMove # @rawPerfFileForCopy # # @return : # 0 if Success # 1 if Error sub deleteRawPerfData { # This variable is used to store the return value of function calls. my $retVal = 0; # We need to rename the folder in /perf folder from "_" to "_.DONE" so that Java # Performance code can use it. It is to avoid the case that Perl script is in the middle of updating "_" folder and Java # code start using it. my @files = glob($eternusProbeFolder."/perf/*"); foreach my $file (@files) { if ($file =~ /(\d\d\d\d\d\d\d\d)_(\d\d\d\d\d\d)$/) { # Genearate the new folder. my $newFolderName = $file.".DONE"; # TODO: Check the return value. rename($file, $newFolderName); logMsg("Info: Renamed [$file] to [$newFolderName]."); } } # Delete file present in the Temporary Raw perf data folder. @files = glob($tempRawPerfDataPath."/*"); foreach my $file (@files) { logMsg("Info: Deleting file: [$file]."); if (1 != unlink($file)) { logMsg("Error: File delete failed. File: [$file]."); $retVal = 1; goto EXIT; } } # Clear the list for files which was used in the last MARS Perf cycle. logMsg("Info: Clearing variables used in current Perl collection cycle."); # Clear the data structures used to store Performance value of last iteration. my $resourcePerfHash = (); my $resourceInstancePerfHash = (); my $counterPerfHash = (); # Loop through the list of all Resource types whose perf data was collected. for my $resourceType (keys %perfData) { $resourcePerfHash = $perfData{$resourceType}; # Loop through the list of Perf counters for the current Resource type. for my $counterName (keys %$resourcePerfHash) { $counterPerfHash = $resourcePerfHash->{$counterName}; # Loop through the list of all Resource instances for the current Perf counter. for my $resourceInstance (keys %$counterPerfHash) { $resourceInstancePerfHash = $counterPerfHash->{$resourceInstance}; # Clear the hash. %$resourceInstancePerfHash = (); } # Clear the hash. %$counterPerfHash = (); } # Clear the hash. %$resourcePerfHash =(); } # Clearing all data structures which are used in the current iteration for collecting the performace data. %perfData =(); @rawPerfFileForMove = (); @rawPerfFileForCopy = (); %usedFileNameList = (); $timeTrace = ""; $selectedFolderForPerfData = ""; $pmdFilesForPerfData = ""; EXIT: return $retVal; } # This function is used to find the Eternus Perf file name which corresponds to the input timestamp. The input timestamp has the format of "hhmmss". # So it could be the case that the raw perf data folder contains multiple files from different dates with same hhmmss level timestamp. So this # function finds the oldest file present in the raw perf data folder having the input timestamp. After finding the correct file, it also updates a # global hash which contains the list of Eternus Perf files from which we are collecting perf data. # # @param : # $_[0] - Timestamp whose corresponding pmd file needs to be serached. # $_[1] - Perf folder name in which pmd file needs to be searched. # # @affected global variables : # $usedFileNameList # # @return : # 0 if Success # 1 if Error sub storePerfFileName { # This variable is used to store the return value of function calls. my $retVal = 0; # This variable is used to store the partial file name for which all related files need to be searched. my $partialFileName = ""; # This variable is used to store the name of pmd file with the lowest time stamp. my $lowestFileTimestamp = 99999999999999; # Get the input parameter. (my $time, my $perfFolder) = @_; # Check whether we have already stored the name of .pmd file associated with the input timestamp. foreach my $key (keys %usedFileNameList) { # If file associated with the input timestamp already present in the list then exit without doing anything. if ($key =~ /(.+)($time).pmd/) { goto EXIT; } } # Construct the file name with wild-card present in it. It is used to find all files from different days having the same timestamp. if ($time =~ /^(\d\d):(\d\d):(\d\d)$/) { # Construct name as: *hhmmss.pmd $partialFileName = "*".$1.$2.$3.".pmd"; } # Get the complete name of all files present in the raw perf data folder and contain the input timestamp. my @files = glob($tempRawPerfDataPath."/".$partialFileName); # Loop through every file name. foreach my $eternusPerfFile (@files) { # Extract the file name from the complete file path. my $fileName = basename($eternusPerfFile); # Get the complete timestamp from the file name which contain YYYYMMDDhhmmss as the timestamp. if ($fileName =~ /^(\d+).pmd$/) { # If current file timestamp is lower than the previously found lowest timestamp then make it lowest timestamp. if ($1 < $lowestFileTimestamp) { $lowestFileTimestamp = $1; } } } # If we are not able to find the correct file then it is an error. if (99999999999999 == $lowestFileTimestamp) { logMsg("Error: Not able to find .pmd file for the timestamp: [$time]."); $retVal = 1; } # Now we have the lowest timestamp available in the current folder. So store the name of .pmd file associated with the lowest timestamp in the # global list of .pmd files. This list contain the name of all .pmd files whose data is getting used to generate MARS performance data file. my $finalFileName = $lowestFileTimestamp.".pmd"; if (not exists $usedFileNameList{$finalFileName}) { logMsg("Info: Found Raw Perf file [$perfFolder/$finalFileName]."); $usedFileNameList{$finalFileName} = $perfFolder; } EXIT: return $retVal; } # This function is used to dump final Performance data in MARS format. # # @affected global variables : # %selectedFolderForPerfData # $PERF_FILE # # @return : # 0 if Success # 1 if Error sub dumpPerfData { # This variable is used to store the return value of function calls. my $retVal = 0; # This variable is used to store the complete path of folder where perf data for the current cycle will be stored. my $marsCurrentPerfFolder = ""; # We need to create the MARS Perf folder which contains the Perf files for the current cycle. This folder name should be the timestamp of first # data point of the current cycle. So sort the list of all pmd files which are used to generate the Performance data in MARS format for the # current cycle, find the timestamp of first pmd file, and make the MARS Perf folder. for my $key (sort {$a<=>$b} keys %usedFileNameList) { if ($key =~ /(\d\d\d\d\d\d\d\d)(\d\d\d\d\d\d).pmd/) { $marsCurrentPerfFolder = $eternusProbeFolder."/perf/$1"."_".$2; logMsg("Info: Creating Folder: [$marsCurrentPerfFolder]."); unless (defined eval {mkpath($marsCurrentPerfFolder)}) { logMsg("Error: Not able to create Perf date folder."); $retVal = 1; goto EXIT; } # Break the loop after creating the Perf folder. last; } } my $resourcePerfHash = (); my $resourceInstancePerfHash = (); my $counterPerfHash = (); my $perfVal = 0; logMsg("Info: Start processing of final Perf data for current cycle."); # Loop through the list of all resource whose Perf data is available. for my $resourceType (keys %perfData) { # This variable is used to store the complete path of folder where current resource specific data will be stored. my $resourceFolder = $marsCurrentPerfFolder."/".$resourceNameHash{$resourceType}; # Create the folder with Resource name inside the MARS Perf folder for the current cycle. unless (defined eval {mkpath($resourceFolder)}) { logMsg("Error: Not able to create Perf Resource folder. Name = [$resourceFolder]."); $retVal = 1; goto EXIT; } $resourcePerfHash = $perfData{$resourceType}; # Loop through the list of Perf counters for the current Resource. for my $counterName (keys %$resourcePerfHash) { # Create the Perf file for the current counter. my $perfFile = $resourceFolder."/".$counterName.".txt"; open($PERF_FILE, ">", $perfFile ) or die logMsg("Error: Not able to open the file: [$perfFile]."); # Dump the Perf metadata in the counter Perf file. prepareProbeMetadata(); $counterPerfHash = $resourcePerfHash->{$counterName}; # Loop thourgh the list of all Resource instances in the current Perf counter. for my $resourceInstance (keys %$counterPerfHash) { $resourceInstancePerfHash = $counterPerfHash->{$resourceInstance}; # This variable is used to construct the comma separated list of Perf data point for the current cycle. my $perfDataPoints = ""; # This variable is used to decide whether we encountered the first Perf data point or not. my $isFirst = 1; # This is used to extract minutes of the timestamp whose perfval entry we have done. my $oldTimeVal = ""; # This is used to extract minutes of the timestamp whose perfval entry we have to do. my $newTimeVal = ""; # This is used to store the timestamp of last pmd file available. If some pmd file(s) are missing after the last pmd file available # then we can make their corresponding entry in the final perf data. my $lastTimeStamp; # This variable is used to keep track of Perf data points processed so far. my $countFoundDataPoint = 0; # This is used to count available Perf data points. my $countAvailableDataPoint = keys %$resourceInstancePerfHash; # Loop through the list of all Perf data point for the current counter. for my $time (sort {$a<=>$b}keys %$resourceInstancePerfHash) { $countFoundDataPoint++; $perfVal = $resourceInstancePerfHash->{$time}; # If it is the first Perf data point then initialize $perfDataPoints with the first Perf data point. if (1 == $isFirst) { $perfDataPoints = $perfVal; # Store minute part of current timestamp for check in the next iteration. $oldTimeVal = substr($time, 10, 2); $isFirst = 0; } # If we already got the first Perf data point then start concatenating the Perf data points. else { $newTimeVal = substr($time, 10, 2); if ($newTimeVal == ($oldTimeVal + 1)) { $perfDataPoints = $perfDataPoints.",".$perfVal; $oldTimeVal = $newTimeVal; } elsif (("59" == $oldTimeVal) && ("00" == $newTimeVal)) { $perfDataPoints = $perfDataPoints.",".$perfVal; $oldTimeVal = "00"; } # This condition arises when there is some datahole between current and previous pmd file. else { # This variable is used to store the number of missing pmd files between 2 available pmd files. my $missingPmdFile = 0; # If the current time range crosses the hour boundary then add 60 minutes to the final timestamp to find the number of dataholes. if ($oldTimeVal > $newTimeVal) { $missingPmdFile = ($newTimeVal + 60) - $oldTimeVal; } else { $missingPmdFile = $newTimeVal - $oldTimeVal; } # Appending entry for missing files in between. while (0 < $missingPmdFile) { $perfDataPoints = $perfDataPoints.","; $missingPmdFile--; } $perfDataPoints = $perfDataPoints.$perfVal; $oldTimeVal = $newTimeVal; } if ($countFoundDataPoint == $countAvailableDataPoint) { $lastTimeStamp = $time; } } } # This variable keeps tracks of missing file after the last file. my $countIteration = 0; # Here we are extracting value of minutes of global variable timeStamp and last pmd file timeStamp. my $finalTimeStampMin = substr($finalTimeStamp, 10, 2); my $lastTimeStampMin = substr($lastTimeStamp, 10, 2); # If this is the case it means that some files are missing at the end. So get the count of those missing file in countIteration variable. if ($finalTimeStampMin > $lastTimeStampMin) { $countIteration = $finalTimeStampMin - $lastTimeStampMin; } # If the current time range crosses the hour boundary then add 60 minutes to the final timestamp to find the number of dataholes. elsif ($finalTimeStampMin < $lastTimeStampMin) { $countIteration = ($finalTimeStampMin + 60)- $lastTimeStampMin; } # Appending entry for missing files after the last file. while (0 < $countIteration) { $perfDataPoints = $perfDataPoints.","; $countIteration--; } # Dump the data for current Resource instance the counter Perf file. print $PERF_FILE "$resourceInstance".","."$perfDataPoints\n"; } close($PERF_FILE); } } EXIT: return $retVal; } # This function is used to check the "PerfRawData" folder for Eternus raw performance data availability. It checks whether there are at least 15 pmd # files present in one or more Perf folders. If there are 15 or more pmd file available then it sets %selectedFolderForPerfData with the oldest folder # names and count of pmd file present in the folders. # # @param : # $_[0] - [Out] 1 if raw performance data is available. 0 if no raw performance data available. # # @affected global variables : # %selectedFolderForPerfData # # @return : # 0 if Success # 1 if Error sub checkRawPerfDataStatus { # This varialbe is used to store the unsorted list of Perf folder names. We are using hash object to save the directory name so that sorting will # be easy. my %folderHash = (); # This variable is used to store the sorted list of Perf folder names. my @sortedFolderList = (); # Get the complete name of all files/folders present in the base folder for raw performance data. my @files = glob($rawPerfDataBasePath."/*"); # Loop through every object name. foreach my $objName (@files) { # If current object is not a directory then move to next object. if (not -d $objName) { next; } # Extract the object name from the complete object path. my $fileName = basename($objName); # Get the complete timestamp from the object name which contain YYYYMMDDhhmmss as the timestamp. if ($fileName =~ /^(\d\d\d\d\d\d\d\d\d\d\d\d\d\d)$/) { $folderHash{$1} = $1; } } # Make the sorted list of folder as we have to process oldest folder first. for my $key (sort {$a<=>$b} keys %folderHash) { push(@sortedFolderList, $key); } # This variable is used to count the total number of pmd files found so far. my $totalPmdFileCount = 0; # This variable is used to count the number of folders in the sortedFolderList map. my $countFolder = @sortedFolderList; # This variable is used to check whether the current iterated folder is the last folder. my $currentFolder = 0; # This variable is used to check whether sufficient raw perf data is available for processing or not. my $isDataPresentForProcessing = 0; # We need to find minimum number of Perf folders which contain at least 15 pmd files. So loop through the list of all folders present in the base # folder. foreach my $folderName (@sortedFolderList) { $currentFolder++; # Get the list of pmd files present in the current folder. my @files = glob($rawPerfDataBasePath."/".$folderName."/*.pmd"); # This variable is used to count the pmd files found in the currnet Perf folder. my $localPmdFileCount = scalar(grep ($_, @files)); logMsg("Info: Currnet folder: [$folderName]. Files in folder: [$localPmdFileCount]"); # If current folder is active folder and contain minimum number of files which are needed for processing then go ahead. if (($PMD_FILES_FOR_ITERATION <= $localPmdFileCount) && ($currentFolder == $countFolder)) { findOldestFiles($folderName, $PMD_FILES_FOR_ITERATION); # TODO: Can we change the name. $selectedFolderForPerfData = $folderName; $pmdFilesForPerfData = $PMD_FILES_FOR_ITERATION; $isDataPresentForProcessing = 1; goto EXIT; } # If the current folder is not active and has less then 2 pmd files then remove the folder. elsif ((1 >= $localPmdFileCount) && ($currentFolder != $countFolder)) { # Providing the complete path of the folder to be removed. my $folderToRemove = $rawPerfDataBasePath."/".$folderName; # TODO: Check the return value. rmtree($folderToRemove); } # If the current folder is not active and it has some unproceesed pmd files. elsif ((1 < $localPmdFileCount) && ($currentFolder != $countFolder)) { # If current Raw Perf folder contain more files than needed. if ($localPmdFileCount >= $PMD_FILES_FOR_ITERATION) { findOldestFiles($folderName, $PMD_FILES_FOR_ITERATION); $selectedFolderForPerfData = $folderName; $pmdFilesForPerfData = $PMD_FILES_FOR_ITERATION; $isDataPresentForProcessing = 1; goto EXIT; } else { findOldestFiles($folderName, $localPmdFileCount); $selectedFolderForPerfData = $folderName; $pmdFilesForPerfData = $localPmdFileCount; $isDataPresentForProcessing = 1; goto EXIT; } } } EXIT: # If we found sufficient files for processing then move the files in temporary folder for further processing. if (1 == $isDataPresentForProcessing) { $_[0] = 1; moveRawPerfFiles(); } else { $_[0] = 0; } return 0; } # This function is used to find the oldest "$fileCount" pmd files present in the input folder. After finding the files, it updates @rawPerfFileForMove # with the complete name of ($fileCount - 1) files. It also updates @rawPerfFileForCopy with the complete name of dcd file along with the # [$fileCount]th pmd file. # # @param : # $_[0] - [In] Folder in which to find the oldest pmd files. # $_[1] - [In] Number of pmd files which needs to looked in the input folder. # # @affected global variables : # @rawPerfFileForMove # @rawPerfFileForCopy # $timeTrace # # @return : # Nothing sub findOldestFiles { # Read input parameter. (my $folderPath, my $fileCount) = @_; logMsg("Info: Input folder: [$folderPath]."); logMsg("Info: Input file count: [$fileCount]."); # This variable is used to store the timestamp of pmd files as the key element so that we can sort the pmd file based on their timestamp. my %fileMap = (); # Get the complete name of all pmd files present in the input folder. my @files = glob($rawPerfDataBasePath."/".$folderPath."/*.pmd"); # Loop through every pmd file. foreach my $objName (@files) { # Extract the file name from the complete file path. my $fileName = basename($objName); # Get the complete timestamp from the pmd file name which contain YYYYMMDDhhmmss as the timestamp. if ($fileName =~ /^(\d\d\d\d\d\d\d\d\d\d\d\d\d\d)\.pmd$/) { # Save the file timestamp as the hash key so that we can sort pmd files based on timestamp. $fileMap{$1} = $objName; # Found the pmd file so decrement the needed file count. $fileCount = $fileCount - 1; # If we found all needed pmd files then exit from the loop. if (0 == $fileCount) { last; } } } # It is used to keep track of iterations completed so far. my $currentIteration = 0; # It is used for counting number of pmd files in the given folder. my $totalCount = keys %fileMap; # It is used for storing the minute part of the first pmd file. my $firstKeyMin = ""; # Store the complete name of pmd files in the global list. for my $key (sort {$a<=>$b} keys %fileMap) { $currentIteration++; # Store the fifteen minutes incremented timestamp of the first pmd file which is used to calculate the value of timeTrace. if (1 == $currentIteration) { push(@rawPerfFileForMove, $fileMap{$key}); # As we have to select totol 16 continuous pmd file, we need to select all pmd files whose timestamp is not greater than 15 minutes added # in the timestamp of first pmd file. $finalTimeStamp = incrementByMinutes($key, 15); # Here we are initializing timeTrace for the current iteration. It needs to be initialized with the timestamp of first pmd file which we # are processing in the current iteration. $timeTrace = substr($key, 0, 8); $timeTraceCopy = $timeTrace; # Save timestamp upto minute part of the first pmd file. $firstKeyMin = substr($key, 0, 12); } # This is the case when timestamp of first and second pmd file upto minutes is same. elsif ((2 == $currentIteration) && ($firstKeyMin eq substr($key, 0, 12))) { # As we have to select total 16 continuous pmd file and minute part of first 2 pmd files are same, we need to select all pmd files whose # timestamp is not greater than 14 minutes added in the timestamp of first/second pmd file. $finalTimeStamp = incrementByMinutes($key, 14); push(@rawPerfFileForMove, $fileMap{$key}); } elsif ($currentIteration >= 2) { # If the current pmd file is not the last file for the current time range then move it to the temporary folder. if (substr($finalTimeStamp, 0, 12) > substr($key, 0, 12)) { push(@rawPerfFileForMove, $fileMap{$key}); } # If the current pmd file is the last file for the current time range then copy it to the temporary folder. To check it, compare the # timestamp upto minutes. elsif (substr($finalTimeStamp, 0, 12) eq substr($key, 0, 12)) { push(@rawPerfFileForCopy, $fileMap{$key}); # Exit the loop as we found the last file. last; } } } # Get the complete name of all dcd files present in the input folder. In general, there is only one dcd file present in a folder. @files = glob($rawPerfDataBasePath."/".$folderPath."/*.dcd"); # Loop through every dcd file. foreach my $objName (@files) { # Store the complete name of dcd file in the global list. push(@rawPerfFileForCopy, $objName); } # Dump the list of selected files. foreach my $file (@rawPerfFileForMove) { logMsg("Info: Files for move in temp folder: [$file]."); } foreach my $file (@rawPerfFileForCopy) { logMsg("Info: Files for copy in temp folder: [$file]."); } } # This function is used to move the selected pmd file for the current MARS Perf cycle in the temp folder. It also copies corresponding dcd file in the # temp folder. # # @affected global variables : # None # # @return : # 0 if Success # 1 if Error sub moveRawPerfFiles { # This variable is used to store the return value of function calls. my $retVal = 0; # Move pmd file from the Perf folder to the temp folder. foreach my $file (@rawPerfFileForMove) { my $fileName = basename($file); logMsg("Info: Moving file : [$file]."); if (0 == move($file, $tempRawPerfDataPath."/".$fileName)) { logMsg("Error: File move failed. Source: [$file]. Destination: [$tempRawPerfDataPath."/".$fileName]."); $retVal = 1; goto EXIT; } } # Copy dcd file along with first pmd file from the Perf folder to the temp folder. Reason is that dcd and pmd file will also be required in the # next cycle of MARS data collection if there are additional pmd files present in the input folder. foreach my $file (@rawPerfFileForCopy) { my $fileName = basename($file); logMsg("Info: Copying file : [$file]."); if (0 == copy($file, $tempRawPerfDataPath."/".$fileName)) { logMsg("Error: File copy failed. Source: [$file]. Destination: [$tempRawPerfDataPath."/".$fileName]."); $retVal = 1; goto EXIT; } } EXIT: return $retVal; } # This function is used to run a shell script to get Eternus Storage name. The script connects to the Eternus Storage and dump the configuration data # in a file. It also verifies the data retrieved from the Eternus Storage. # # @affected global variables : # $eternusStorageName # # @return : # 0 if Success # 1 if Error sub getEternusStorageNameSsh { # This variable is used to store the return value of function calls. my $retVal = 0; # This variable is used to store the exit status of "system" command. my $exitCode = 0; # It is used to check whether we found correct information in the Verification dump file. my $found = 0; # This variable is used to store the command to get the Eternus conf data. my $cmd = ""; # Construct the command to get the Eternus Conf data. $cmd = $eternusRunTimeLocation."/eternusGetConfData.sh System $eternusIpAddress $eternusConfUserId $eternusConfPassword > $confDumpFile"; logMsg("Info: Running command: [$cmd]."); # Run the command to get the Eternus Conf data. $exitCode = system($cmd); if (0 != $exitCode) { logMsg("Error: Failed to execute: [eternusGetConfData.sh]."); $retVal = 1; goto EXIT; } logMsg("Info: Executed Shell script for connecting to Eternus Storage."); logMsg("Info: Opening Eternus Storage verification dump file. File = [$confDumpFile]."); open(CONF_DUMP, $confDumpFile) or die logMsg("Error: Not able to open the file: [$confDumpFile]."); # Loop through the content of Verification dump file. while () { # Remove the new line character. chomp; logMsg("Info: Eternus Storage connection data = [$_]."); # Look for "Summary Status" string in the file. If we found that then it means that we can connect to Eternus Storage. if (/^(\s*)Name(\s*)\[(.+)\](\s*)$/) { $eternusStorageName = $3; logMsg("Info: Eternus Storage name: [$eternusStorageName]."); $found = 1; last; } } if (0 == $found) { $retVal = 1; } close(CONF_DUMP); EXIT: return $retVal; } # This function is used to run a MICC command to get Eternus Storage name. The command connects to the Eternus Storage and dump the configuration data # in a file. It also verifies the data retrieved from the Eternus Storage. # # @affected global variables : # $eternusStorageName # # @return : # 0 if Success # 1 if Error sub getEternusStorageNameMicc { # This variable is used to store the return value of function calls. my $retVal = 0; # This variable is used to store the exit status of "system" command. my $exitCode = 0; # It is used to check whether we found correct information in the Verification dump file. my $found = 0; # This variable is used to store the command to get the Eternus conf data. my $cmd = ""; # Construct the command to get the Eternus Conf data. $cmd = $miccUtil." DevSts -i ".$eternusIpAddress." -v > $confDumpFile"; logMsg("Info: Running command: [$cmd]."); # Run the command to get the Eternus Conf data. $exitCode = system($cmd); if (0 != $exitCode) { logMsg("Error: Failed to execute: [$cmd]."); $retVal = 1; goto EXIT; } logMsg("Info: Executed command for connecting to Eternus Storage."); logMsg("Info: Opening Eternus Storage verification dump file. File = [$confDumpFile]."); open(CONF_DUMP, $confDumpFile) or die logMsg("Error: Not able to open the file: [$confDumpFile]."); # Loop through the content of Verification dump file. while () { # Remove the new line character. chomp; logMsg("Info: Eternus Storage connection data = [$_]."); # Get "name" property. if (/^(\s*)\|(\s*)Machine Name(\s*)\|(\s*)(.+)(\s*)\|/) { $eternusStorageName = $5; # Remove trailing whitespaces from the end of $eternusStorageName. $eternusStorageName =~ s/\s+$//; # If machine name not present then replace it with IP address. if ("" eq $eternusStorageName) { $eternusStorageName = $eternusIpAddress; } logMsg("Info: Eternus Storage name: [$eternusStorageName]."); $found = 1; last; } } if (0 == $found) { $retVal = 1; } close(CONF_DUMP); EXIT: return $retVal; } # This is used to increment a given timestamp in 'YYYYMMDDHHMMSS' format by 15 minutes. # # @affected global variables : # None # # @param : # $_[0] - [In] TimeStamp which need to be incremented by fifteen minutes. # $_[1] - [In] Number of minutes which needs to be incremented in the input timestamp. # # @return : # Incremented timestamp. sub incrementByMinutes { # Get the input parameters. (my $timestamp, my $minToIncrement) = @_; # Extract year, month, day, hour, minutes, seconds from the input timestamp value. my $sec = substr($timestamp, 12, 2); my $min = substr($timestamp, 10, 2); my $hour = substr($timestamp, 8, 2); my $day = substr($timestamp, 6, 2); my $month = substr($timestamp, 4, 2); my $year = substr($timestamp, 0, 4); # It is done because the function timelocal considers month in the range of 0 to 11. $month = $month - 1; # This function converts the given time in terms of seconds. my $time = timelocal($sec, $min, $hour, $day, $month, $year); # Incremented fifteen minutes to the given time. $time = $time + ($minToIncrement * 60); # Getting the incremented values. ($sec, $min, $hour, $day, $month, $year, my $wday, my $yday, my $isdst) = localtime($time); # It is done because in the year parameter it supplies the difference between incremented year and 1900. $year = 1900 + $year; # The actual month value will be one more than that incremented. $month = $month + 1; # This is done to convert each of sec, min, hour, day, month to two digits so that we get proper timestamp. if (!($sec =~ /[0-9][0-9]/)) { $sec = "0$sec"; } if (!($min =~ /[0-9][0-9]/)) { $min = "0$min"; } if (!($hour =~ /[0-9][0-9]/)) { $hour = "0$hour"; } if (!($day =~ /[0-9][0-9]/)) { $day = "0$day"; } if (!($month =~ /[0-9][0-9]/)) { $month = "0$month"; } # Return the incremented timestamp in proper formatt of yyyymmddhhmmss. return $year.$month.$day.$hour.$min.$sec ; } # This function is used to update the MARS counter performance file with the Metadata information about the Probe. # # @affected global variables : # None # # @return : # Nothing sub prepareProbeMetadata { print $PERF_FILE "%meta\n"; print $PERF_FILE "probe_id:$marsSiteName"."_"."$eternusIpAddress\n"; print $PERF_FILE "probe_type:Eternus\n"; print $PERF_FILE "probe_site:$marsSiteName\n"; print $PERF_FILE "probe_server:$eternusIpAddress\n"; print $PERF_FILE "probe_version:$probeVersion\n"; print $PERF_FILE "collection_interval:60\n"; print $PERF_FILE "number_of_samples:15\n"; print $PERF_FILE "%meta\n"; } # This function is used to log a message in the log file. It logs the message in the following format: # [YYYY-MM-DD HH:MM:ss,SSS] [Eternus-] [Perf] # # @param : # $_[0] - Log message. # # @affected global variables : # None # # @return : # Nothing sub logMsg { # This variable is used to store the complete path of probe specific folder in db/probe folder. my $eternusLogFolder = ""; # Set Eternus log folder path. $eternusLogFolder = $baseFolder."/logs/eternusProbe"; # If log folder does not exist then create it. if (!(-e $eternusLogFolder)) { # TODO: Check the return value of mkpath. mkpath($eternusLogFolder); } # Get the current date and time. my ($seconds, $microseconds) = gettimeofday; my $currentTime = strftime("%Y-%m-%d %H:%M:%S", localtime($seconds)); # Get the date from the given date and time . my($date, $time) = split(' ', $currentTime, 2); # Set the name of log file. $logFile = $eternusLogFolder."/eternusProbe.".$date.".log"; # Open the log file. open($LOG_HANDLE, ">>", $logFile) or die "Not able to open the file: [$logFile]."; # Read input parameters. my $logMsg = $_[0]; # For getting time in a given format. $currentTime = "[".$currentTime.",".$microseconds."]"; # Construct the final log message. my $finalMsg = $currentTime." [Eternus-".$eternusIpAddress."] [Perf] ".$logMsg."\n"; # Dump the final log message in log file. print $LOG_HANDLE $finalMsg; } # This function is used to de-initialize the environment. # # @affected global variables : # $CONF_HANDLE # # @return : # None sub deInit { close($LOG_HANDLE); }