#!/usr/bin/perl -w
##
# @file xflowPerfDataProcessor.pl
# @brief Generates raw format Flow files which are later used to generate MARS format perf and conf files.
# @author Manojkumar M Waghmare
# @version 1.0
# @date 2013-07-17
#
use strict; 
use Getopt::Std;
use File::Basename;
use File::Path;

use Time::HiRes qw(gettimeofday); #for current date and time.
use POSIX;
use POSIX "strftime";

use Date::Calc;
use Data::Dumper;

###########################################################
# Subroutines Forward Declarations
##########################################################{#
sub getNextDay;
sub getPrevDay;
sub getNthDay;
sub getNextInterval;
sub logMsg;
#}##########################################################

###########################################################
# Globals & Constants
##########################################################{#
my $NFDUMP="/usr/local/bin/nfdump";

if( !-e $NFDUMP ) {
	print STDERR "\nnfdump binary $NFDUMP not found.\n";
	exit 1;
}

#use constant MEGHA_DEPLOY_DIR	=>	/usr/local/megha
use constant MEGHA_DEPLOY_DIR	=>	"/usr/local/megha";
use constant MEGHA_LOG_DIR		=>	&MEGHA_DEPLOY_DIR . "/logs";
use constant XFLOW_RUNTIME_DIR	=>	&MEGHA_DEPLOY_DIR . "/lib/xflow";
my $GENPERF_SCRIPT = &XFLOW_RUNTIME_DIR . "/generatePerf.pl";

#Max time difference between network device and probe should not be more than 5 minutes.	
use constant MAX_DEVICE_PROBE_TIME_DIFF => 5;   

#Max time by which an xFlow datagram may get delayed. 
#This should always be greater than MAX_DEVICE_PROBE_TIME_DIFF
use constant MAX_FLOWPACKET_DELAY_TIME	=> 10;	

#Max no. of days for which the nfcapd/sfcapd generated binary flow files are preserved.
use constant MAX_EXPIRY_INTERVAL_DAYS 	=> 2;	

use constant NFDUMP_OUT_FORMAT => '"fmt:%pr,%sp,%dp,%sa,%da,%byt,%sas,%das,%svln,%dvln"';

#constants for command line switches.
use constant {
	AGGR_INTERVAL	=> 'a',
	BINFILES_DIR	=> 'd',
	START_TIME		=> 's',
	END_TIME		=> 'e',
	PROBE_VERSION	=> 'v',
	DEVICE_IP		=> 'i',
	DEVICE_NAME		=> 'n',
	PROBE_SITENAME	=> 'p',
	WORKSPACE_DIR	=> 'w'
};

my %CmdLineArgsDesc = ( 
	&AGGR_INTERVAL	=> "Aggregation Interval in Minutes",
	&BINFILES_DIR	=> "Directory Containing nfcapd Binary Flow Files",
	&START_TIME		=> "Start Time",
	&END_TIME		=> "End Time",
	&PROBE_VERSION	=> "Version of xFlow Probe",
	&DEVICE_IP		=> "IP of the xFlow Device",
	&DEVICE_NAME	=> "Name of the xFlow Device",
	&PROBE_SITENAME	=> "Probe Site Name",	
	&WORKSPACE_DIR	=> "Workspace directory for script"
);

use constant { 
	DEBUG      =>  "<DEBUG>",
    INFO       =>  "<INFO>",
    WARNING    =>  "<WARNING>",
    ERROR      =>  "<ERROR>",
    FATAL      =>  "<FATAL>"   
};

my %CmdLineArgs;

my $Usage = "\n$0 ";
foreach my $key (sort keys %CmdLineArgsDesc) {
	$Usage = $Usage . " -$key <$CmdLineArgsDesc{$key}>";	
}
$Usage = $Usage . " [-h]\n";
#}##########################################################


###########################################################
# MAIN Program
##########################################################{#
getopts('hd:a:s:e:v:i:n:p:w:', \%CmdLineArgs);

#if help requested i.e. -h switch is present 
if( $CmdLineArgs{h} ) {
	print $Usage;
	exit 0;
}

#if any of the required options are missing
if(keys %CmdLineArgs < keys %CmdLineArgsDesc) {
	print STDERR "\nOne or more required options are missing.\n";
	print STDERR $Usage;
	exit 0;
}

my $WorkSpaceDir = $CmdLineArgs{&WORKSPACE_DIR};
if( !-d $WorkSpaceDir ) {
	print STDERR "Workspace Directory: $WorkSpaceDir does NOT exist.";
	exit 1;
}

my $XFlowDeviceName = $CmdLineArgs{&DEVICE_NAME};
my $XFlowDeviceIP = $CmdLineArgs{&DEVICE_IP};

my ($seconds, $microseconds) = gettimeofday;
my $LOGDay = strftime("%Y%m%d", localtime($seconds));
#my $LOGFile = $WorkSpaceDir . "/xflowDataProcessor_$LOGDay". ".log";
my $LOGFile = &MEGHA_LOG_DIR . "/xflowDataProcessor_$LOGDay". ".log";

my $LOGFILE_HANDLE;
open($LOGFILE_HANDLE, ">>", $LOGFile) or die "[FATAL ERROR]: Failed to open the LOG file: [$LOGFile].";

my $msg = "\nCommand line arguments passed - ";
foreach my $switch (sort keys(%CmdLineArgs)) {
	$msg = $msg . "\n$CmdLineArgsDesc{$switch}: $CmdLineArgs{$switch}";
}
logMsg(INFO, $msg);

#check aggregation interval.
my $AggregationInterval = $CmdLineArgs{&AGGR_INTERVAL};
if( 0 == $AggregationInterval ) {
	logMsg(FATAL, "AggregationInterval must be non zero.");
	exit 1;
}

#Find out number extra samples that need to be processed.
my $ExtraSamples = ceil(&MAX_FLOWPACKET_DELAY_TIME/$AggregationInterval);
my $PreviousSamples = ceil(&MAX_DEVICE_PROBE_TIME_DIFF/$AggregationInterval);
logMsg(DEBUG, "Number of Extra Samples to be processed : $ExtraSamples");
logMsg(DEBUG, "Number of Previous Samples to be processed : $PreviousSamples");

#Base directory containing nfcapd binary files for the device
my $BinFilesBaseDir = $CmdLineArgs{&BINFILES_DIR};
if( !-d $BinFilesBaseDir ) {
	logMsg(FATAL, "Directory: $BinFilesBaseDir does NOT exist.");
	exit 1;
}

my $TempDir = $WorkSpaceDir . "/Temp"; 
if( !-d $TempDir ) {
	my @mkdircmd = ("mkdir", $TempDir);
	if( system(@mkdircmd) != 0 ) {
		logMsg(FATAL, "Failed to create Temp directory in workspace dir: $WorkSpaceDir");
		exit 1;
	}
}

my $PerfDir = $WorkSpaceDir . "/perf";
if( !-d $PerfDir ) {
	my @mkperfdircmd = ("mkdir", $PerfDir);
	if( system(@mkperfdircmd) != 0 ) {
		logMsg(FATAL, "Failed to create perf directory in workspace dir: $WorkSpaceDir");
		exit 1;
	}
}

#Find subdirectory in base dir that contains files for given time interval.
my $ST = $CmdLineArgs{&START_TIME};
my $ET = $CmdLineArgs{&END_TIME};

my $TODAY = strftime("%Y-%m-%d", localtime($ST));
my $TODAY_YYYYMMDD = strftime("%Y%m%d", localtime($ST));
my $TODAY_SLASHES = strftime("%Y/%m/%d", localtime($ST));

my ($expireDay, $expireDay_YYYYMMDD) = getNthDay($TODAY, -&MAX_EXPIRY_INTERVAL_DAYS);
my $expireDir = $BinFilesBaseDir . "/" . $expireDay;
if(-d $expireDir) { #if dir which is supposed to be expired exists then delete it.
	logMsg(DEBUG, "Expiring raw files dir: $expireDir");
	my @CMD_rmdirExpDir = ("rm", "-rf", "$expireDir");
	if(system(@CMD_rmdirExpDir) != 0) { 
		logMsg(ERROR, "Failed to expire raw files dir: $expireDir");
	}
}

my $CurrentBinaryDataDir = $BinFilesBaseDir . "/" . $TODAY;
if( !-d $CurrentBinaryDataDir ) {
	logMsg(FATAL, "Nfcapd binary files Directory: $CurrentBinaryDataDir does NOT exist.");
	exit 1;
}

#####################################################################
# List out binary files to be processed in the given time interval.
###################################################################{##
my $StartTime = strftime("%H:%M", localtime($ST));
my $EndTime = strftime("%H:%M", localtime($ET));

my ($startTimeHour, $startTimeMinute) = split(/:/,$StartTime, 2);
my ($endTimeHour, $endTimeMinute) = split(/:/,$EndTime, 2);

my $ExportInterval = (($endTimeMinute == 0) ? 60 : $endTimeMinute) - $startTimeMinute;
my $NumberOfSamples = $ExportInterval / $AggregationInterval;
logMsg(INFO, "Number of Samples in the given interval [$StartTime - $EndTime] : $NumberOfSamples");

my $BinFileBase = $CurrentBinaryDataDir . "/nfcapd.$TODAY_YYYYMMDD";
my $Minute = 0 + $startTimeMinute;

my @BinFiles = ();
my @AIntervals = ();
my $FirstNonEmptyBinFileIndex = -1;
my $index = 0;
for(1 .. $NumberOfSamples) {
	if( $Minute < 10 ) { $Minute = "0$Minute"; }
	push(@AIntervals, $Minute);

	my $BinFile = $BinFileBase . $startTimeHour . $Minute;
	if(!-e $BinFile) {  #in case if bin file does not exist because probe started collection.
		logMsg(DEBUG, "Binary file $BinFile does not exist. Creating one.");
		my @CmdTouch = ("touch", "$BinFile");
		if(system(@CmdTouch) != 0) {
			logMsg(ERROR, "Failed to create binary file: $BinFile");
		}
	}
	else {
		if($FirstNonEmptyBinFileIndex == -1) {
			$FirstNonEmptyBinFileIndex = $index;
		}
	}
	$index++;
	push(@BinFiles, $BinFile);	
	$Minute += $AggregationInterval;
}
push(@AIntervals, $Minute);

if($FirstNonEmptyBinFileIndex == -1) {
	logMsg(FATAL, "None of the binary file in the given time period exists.");
	exit 1;
}

#}####################################################################
# List out Next Sample Files to be processed to accomodate -
# i) Delay in xFlow datagram arrival at collector. 
# ii) Time difference between device and probe appliance.
####################################################################{#
my @NextBinFiles = ();
if($endTimeHour == 0 and $endTimeMinute == 0) { #if it is end of today.
	logMsg(DEBUG, "##### It's end of the day ######");
	my ($TOMORROW, $TOMORROW_YYYYMMDD) = getNextDay($TODAY);

	my $NextBinaryDataDir = $BinFilesBaseDir . "/" . $TOMORROW;
	if( -d $NextBinaryDataDir ) {
		my $NextBinFileBase = $NextBinaryDataDir . "/nfcapd.$TOMORROW_YYYYMMDD";
		my ($eh, $em) = ($endTimeHour, $endTimeMinute);

		for(1 .. $ExtraSamples) {
			my $NextBinFile = $NextBinFileBase. $eh . $em;
			push(@NextBinFiles, $NextBinFile);
			if(! -e $NextBinFile ) {
				logMsg(FATAL, "Next File: $NextBinFile does NOT exist.");
			}

			$em += $AggregationInterval;
			$em = "0$em" if($em < 10);
		}
	}
	else {
		logMsg(FATAL, "Next Day's Data Dir: $NextBinaryDataDir does NOT exist.");
	}
}
else {
	my ($eh, $em) = ($endTimeHour, $endTimeMinute);

	for(1 .. $ExtraSamples) {
		my $NextBinFile = $BinFileBase . $eh . $em;
		push(@NextBinFiles, $NextBinFile);
		if(! -e $NextBinFile ) {
			logMsg(FATAL, "Next File: $NextBinFile does NOT exist.");
		}

		$em += $AggregationInterval;
		$em = "0$em" if($em < 10);
	}
}

#}####################################################################
# List out Prev Backlog Files to accomodate the
# time difference between device and probe appliance.
####################################################################{#
my @PrevBinFiles = ();
if($startTimeHour == 0 and $startTimeMinute == 0) { #if its beginnig of the day.
	logMsg(DEBUG, "##### It's beginning of the day ######");
	my ($YESTERDAY, $YESTERDAY_YYYYMMDD) = getPrevDay($TODAY);

	my $PrevBinaryDataDir = $BinFilesBaseDir . "/" . $YESTERDAY;
	if( -d $PrevBinaryDataDir ) {
		my $PrevBinFileBase = $PrevBinaryDataDir . "/nfcapd.$YESTERDAY_YYYYMMDD";
		my ($sh, $sm) = (23, 60);

		for(1 .. $PreviousSamples) {
			$sm -= $AggregationInterval;
			if($sm < 10) { $sm = "0$sm"; }

			my $PrevBinFile = $PrevBinFileBase. $sh . $sm;
			if( -e $PrevBinFile ) {
				push(@PrevBinFiles, $PrevBinFile);
			}
			else {
				logMsg(WARNING, "Previous File: $PrevBinFile does NOT exist.");
			}
		} #for
	}#if
	else {
		logMsg(WARNING, "Previous Day's Data Dir: $PrevBinaryDataDir does NOT exist.");
	}
}
else {
	my ($sh, $sm) = ($startTimeHour, $startTimeMinute);
	if( $sm == 0 ) { #if its beginning of an hour
		$sm = 60;
		$sh -= 1;
		$sh = "0$sh" if($sh < 10);
	}

	for(1 .. $PreviousSamples) {
		$sm -= $AggregationInterval;
		if($sm < 10) { $sm = "0$sm"; }

		my $PrevBinFile = $BinFileBase . $sh . $sm;
		#Prev file may or may not exist depending on when probe started collection.
		if( -e $PrevBinFile ) { 
			push(@PrevBinFiles, $PrevBinFile);
		}
	}
}
@PrevBinFiles = sort @PrevBinFiles;

logMsg(DEBUG, "####### Number of Files to Process: " . scalar @BinFiles);
logMsg(DEBUG, Dumper(\@BinFiles));

logMsg(DEBUG, "####### Number of Prev Files to Process: " . scalar @PrevBinFiles);
logMsg(DEBUG, Dumper(\@PrevBinFiles));

logMsg(DEBUG, "####### Number of Next Files to Process: " . scalar @NextBinFiles);
logMsg(DEBUG, Dumper(\@NextBinFiles));

#}###############################################################
# Generate RAW Text Files required for creating MARS perf files.
###############################################################{#

my $RawFilesDir = $TempDir . "/" . $TODAY_YYYYMMDD . "_" . $startTimeHour . $AIntervals[0] . "00";
my @CMD_rmdirRawFilesDir = ("rm", "-rf", "$RawFilesDir");
if(-d $RawFilesDir) {
	if(system(@CMD_rmdirRawFilesDir) != 0) { 
		logMsg(ERROR, "Failed to delete raw files dir: $RawFilesDir");
	}
}

my @CMD_mkdirRawFilesDir = ("mkdir", "$RawFilesDir");
if(system(@CMD_mkdirRawFilesDir) != 0) { 
	logMsg(FATAL, "Failed to Create Raw Files Dir: $RawFilesDir");
	exit 1;
}

my $idx = 0;
my @RawFiles=();
for my $BinFile (@BinFiles) {
	logMsg(DEBUG, "Processing Binary File: $BinFile");
	my $fileName = basename($BinFile);
	my $intervalStart = "$TODAY_SLASHES.$startTimeHour:$AIntervals[$idx]:00";
	my $intervalEnd = "$TODAY_SLASHES.$startTimeHour:$AIntervals[$idx+1]:00";
	if( 60 == $AIntervals[$idx + 1] ) {
		$intervalEnd = getNextInterval($intervalStart);
	}
	
	my $timeWindow = "$intervalStart-$intervalEnd";
	$idx++;

	my $OutFile = $RawFilesDir . "/$fileName.raw";
	my $firstFile = (scalar @PrevBinFiles) ? $PrevBinFiles[0] : $BinFiles[$FirstNonEmptyBinFileIndex];
	my $lastFile = basename($NextBinFiles[$#NextBinFiles]);
	my $nfdumpcmd = "$NFDUMP -q -a -R $firstFile:$lastFile -t $timeWindow -o " . &NFDUMP_OUT_FORMAT . " > $OutFile";

	logMsg(DEBUG, "Executing Command: $nfdumpcmd");

	system($nfdumpcmd);
	my $CmdStatus = $?;
	if( $CmdStatus != 0 ) {
		logMsg(ERROR, "Raw File Generation Failed for Time Window: $timeWindow");
		system("touch $OutFile");
	}
	push(@RawFiles, $OutFile);
}

#my $GENPERF_SCRIPT="/root/code/work/xFlowProbe/generatePerf.pl";
my $numSamples = scalar @RawFiles;

#./generatePerf.pl -a 1 -d /tmp/xflow/Temp/20130717_234500 -i 192.168.20.66 -n vyatta -o /tmp/perf/ -p Admin -s 15 -v 10
my $GenPerfCmd = "perl $GENPERF_SCRIPT -a $AggregationInterval -d $RawFilesDir -s $numSamples -o $PerfDir -n $XFlowDeviceName -i $XFlowDeviceIP -p $CmdLineArgs{&PROBE_SITENAME} -v $CmdLineArgs{&PROBE_VERSION}";

logMsg(DEBUG, "Executing Command: $GenPerfCmd");

system($GenPerfCmd);
my $genPerfCmdStatus = $?;
if( $genPerfCmdStatus != 0 ) {
	logMsg(ERROR, "MARS format Perf Files Generation Failed For: $RawFilesDir");
	exit 1;
}

if(system(@CMD_rmdirRawFilesDir) != 0) { 
	logMsg(ERROR, "Failed to delete raw files dir: $RawFilesDir");
}

logMsg(INFO, "Done!!!");
close($LOGFILE_HANDLE);
exit 0;

#}##########################################################################
#}##########################################################################

sub getNextInterval
{
	my ($curInterval) = @_;
	#2013/07/17.23:56:00
	my @tvals = ($curInterval =~ /(\d{4})\/(\d{2})\/(\d{2})\.(\d{2}):(\d{2}):(\d{2})/);
	my @nvals = Date::Calc::Add_Delta_YMDHMS($tvals[0],$tvals[1],$tvals[2],$tvals[3],$tvals[4],$tvals[5], 0,0,0,0,$AggregationInterval,0);

	$nvals[1] = "0$nvals[1]" if($nvals[1] < 10);
	$nvals[2] = "0$nvals[2]" if($nvals[2] < 10);
	$nvals[3] = "0$nvals[3]" if($nvals[3] < 10);
	$nvals[4] = "0$nvals[4]" if($nvals[4] < 10);
	$nvals[5] = "0$nvals[5]" if($nvals[5] < 10);

	my $nextInterval= "$nvals[0]/$nvals[1]/$nvals[2].$nvals[3]:$nvals[4]:$nvals[5]";
	return $nextInterval;
}

sub getNextDay
{
	my ($today_yyyymmdd) = @_;
	my @tvals = ( $today_yyyymmdd =~ /(\d{4})-(\d{2})-(\d{2})/ );
	my @nvals = Date::Calc::Add_Delta_YMD($tvals[0],$tvals[1],$tvals[2],0,0,1);

	$nvals[1] = "0$nvals[1]" if($nvals[1] < 10);
	$nvals[2] = "0$nvals[2]" if($nvals[2] < 10);

	my $tomorrow = "$nvals[0]$nvals[1]$nvals[2]";
	my $tomorrow_yyyymmdd = "$nvals[0]-$nvals[1]-$nvals[2]";
	return ($tomorrow_yyyymmdd, $tomorrow);
}

sub getPrevDay
{
	my ($today_yyyymmdd) = @_;
	my @tvals = ( $today_yyyymmdd =~ /(\d{4})-(\d{2})-(\d{2})/ );
	my @nvals = Date::Calc::Add_Delta_YMD($tvals[0],$tvals[1],$tvals[2],0,0,-1);

	$nvals[1] = "0$nvals[1]" if($nvals[1] < 10);
	$nvals[2] = "0$nvals[2]" if($nvals[2] < 10);

	my $yesterday = "$nvals[0]$nvals[1]$nvals[2]";
	my $yesterday_yyyymmdd = "$nvals[0]-$nvals[1]-$nvals[2]";
	return ($yesterday_yyyymmdd, $yesterday);
}

sub getNthDay
{
	my ($refday_yyyymmdd, $offset) = @_;
	my @tvals = ( $refday_yyyymmdd =~ /(\d{4})-(\d{2})-(\d{2})/ );
	my @nvals = Date::Calc::Add_Delta_YMD($tvals[0],$tvals[1],$tvals[2],0,0,$offset);

	$nvals[1] = "0$nvals[1]" if($nvals[1] < 10);
	$nvals[2] = "0$nvals[2]" if($nvals[2] < 10);

	my $nThDay = "$nvals[0]$nvals[1]$nvals[2]";
	my $nThDay_yyyymmdd = "$nvals[0]-$nvals[1]-$nvals[2]";
	return ($nThDay_yyyymmdd, $nThDay);
}

###
# Logs a message to the log file in the following format:
# [YYYY-MM-DD HH:MM:ss] <INFO|DEBUG|WARNING|ERROR|FATAL> [DeviceName] File(Line): <Log message>
sub logMsg
{
	my( $file, $line )= ( caller )[1,2];
	my($loglevel, $msg) = @_;

	if(	($loglevel eq FATAL) or ($loglevel eq ERROR) or 
		($loglevel eq DEBUG)) {
		print STDERR "\n$msg\n";
	}

    my ($seconds, $microseconds) = gettimeofday;
    my $currentTime = strftime("%Y-%m-%d %H:%M:%S", localtime($seconds));
    my $LogMsg = "[$currentTime,$microseconds] $loglevel [$XFlowDeviceName] $file($line): $msg\n";

    print $LOGFILE_HANDLE $LogMsg;
}