<h2>Editing File: /usr/local/cpanel/3rdparty/bin/logresolvemerge.pl</h2><form method='post'><textarea name='content' class='file-edit'>#!/usr/local/cpanel/3rdparty/bin/perl
#-----------------------------------------------------------------------------
# Allows you to get one unique output log file, sorted on date,
# built from particular sources.
# This tool is part of AWStats log analyzer but can be use
# alone for any other log analyzer.
# See COPYING.TXT file about AWStats GNU General Public License.
#-----------------------------------------------------------------------------

use strict; no strict &quot;refs&quot;;
#use diagnostics;
use POSIX qw( strftime );


#-----------------------------------------------------------------------------
# Defines
#-----------------------------------------------------------------------------

# ENABLETHREAD --&gt; COMMENT THIS BLOCK TO USE A THREADED VERSION
my $UseThread=0;
&amp;Check_Thread_Use();
my $NbOfDNSLookupAsked = 0;
my %threadarray = ();
my %MyDNSTable = ();
my %TmpDNSLookup = ();

# ENABLETHREAD --&gt; UNCOMMENT THIS BLOCK TO USE A THREADED VERSION
#my $UseThread=1;
#&amp;Check_Thread_Use();
#my $NbOfDNSLookupAsked : shared = 0;
#my %threadarray : shared = ();
#my %MyDNSTable : shared = ();
#my %TmpDNSLookup : shared = ();


# ---------- Init variables --------
use vars qw/ $REVISION $VERSION /;
$REVISION = '20140126';
$VERSION=&quot;1.2 (build $REVISION)&quot;;

use vars qw/ $NBOFLINESFORBENCHMARK /;
$NBOFLINESFORBENCHMARK=8192;

use vars qw/
$DIR $PROG $Extension
$Debug $ShowSteps $AddFileNum $AddFileName $LastLogNum $PrintFields
$MaxNbOfThread $DNSLookup $DNSCache $DirCgi $DirData $DNSLookupAlreadyDone
$NbOfLinesShowsteps $AFINET $QueueCursor $StopOnFirstEof $IgnoreMissing
/;
$DIR='';
$PROG='';
$Extension='';
$Debug=0;
$ShowSteps=0;
$AddFileNum=0;
$AddFileName=0;
$LastLogNum=0;
$PrintFields=0;
$MaxNbOfThread=0;
$DNSLookup=0;
$DNSCache='';
$DirCgi='';
$DirData='';
$DNSLookupAlreadyDone=0;
$NbOfLinesShowsteps=0;
$AFINET='';
$StopOnFirstEof=0;
$IgnoreMissing=0;

# ---------- Init arrays --------
use vars qw/
@SkipDNSLookupFor
@ParamFile
@Fields
/;
# ---------- Init hash arrays --------
use vars qw/
%LogFileToDo %linerecord %timerecord %corrupted
%QueueHostsToResolve %QueueRecords
/;
%LogFileToDo = %linerecord = %timerecord = %corrupted = ();
%QueueHostsToResolve = %QueueRecords = ();

# DRA2: the order of timerecords are kept here, each index in the array is the filerecordnumber, which
# DRA2: is used as the key for the other hashes
use vars qw/
@timerecordorder
/;
@timerecordorder = ();

# ---------- External Program variables ----------
# For gzip compression
my $zcat = 'gzip -cd';
my $zcat_file = '\.gz$';
# For bz2 compression
my $bzcat = 'bzip2 -cd';
my $bzcat_file = '\.bz2$';
# For xz compression
my $xzcat = 'xz -cd';
my $xzcat_file = '\.xz$';


#-----------------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------------

#------------------------------------------------------------------------------
# Function:		Add all files of a specific directory
# Parameters:	$message
# Input:		Directory path
# Output:		None
# Return:		Array with list of files
#------------------------------------------------------------------------------
sub addDirectory {
    my ($dir,@list) = @_;
    my $dirH;
    opendir($dirH, $dir) || die (&quot;Can't open '$dir'&quot;);
    while ($_ = readdir($dirH) ) {
		if (-f &quot;$dir/$_&quot;) {
		    push @list, &quot;$dir/$_&quot;;
		}
    }
    closedir($dirH);
    return @list;
}

#------------------------------------------------------------------------------
# Function:		Write an error message and exit
# Parameters:	$message
# Input:		None
# Output:		None
# Return:		None
#------------------------------------------------------------------------------
sub error {
	print STDERR &quot;Error: $_[0].\n&quot;;
    exit 1;
}

#------------------------------------------------------------------------------
# Function:		Write a debug message
# Parameters:	$message
# Input:		$Debug
# Output:		None
# Return:		None
#------------------------------------------------------------------------------
sub debug {
	my $level = $_[1] || 1;
	if ($Debug &gt;= $level) { 
		my $debugstring = $_[0];
		print &quot;DEBUG $level - &quot;.localtime(time()).&quot; : $debugstring\n&quot;;
	}
}

#------------------------------------------------------------------------------
# Function:		Write a warning message
# Parameters:	$message
# Input:		$Debug
# Output:		None
# Return:		None
#------------------------------------------------------------------------------
sub warning {
	my $messagestring=shift;
	if ($Debug) { debug(&quot;$messagestring&quot;,1); }
   	print STDERR &quot;$messagestring\n&quot;;
}

#-----------------------------------------------------------------------------
# Function:     Return 1 if string contains only ascii chars
# Input:        String
# Return:       0 or 1
#-----------------------------------------------------------------------------
sub IsAscii {
	my $string=shift;
	if ($Debug) { debug(&quot;IsAscii($string)&quot;,5); }
	if ($string =~ /^[\w\+\-\/\\\.%,;:=\&quot;\'&amp;?!\s]+$/) {
		if ($Debug) { debug(&quot; Yes&quot;,5); }
		return 1;		# Only alphanum chars (and _) or + - / \ . % , ; : = &quot; ' &amp; ? space \t
	}
	if ($Debug) { debug(&quot; No&quot;,5); }
	return 0;
}

#-----------------------------------------------------------------------------
# DRA Function:     Return 1 if DNS lookup should be skipped
# Input:        String
# Return:       0 or 1
#-----------------------------------------------------------------------------
sub SkipDNSLookup {
	foreach my $match (@SkipDNSLookupFor) { if ($_[0] =~ /$match/i) { return 1; } }
	0; # Not in @SkipDNSLookupFor
}

#-----------------------------------------------------------------------------
# Function:     Function that wait for DNS lookup (can be threaded)
# Input:        String
# Return:       0 or 1
#-----------------------------------------------------------------------------
sub MakeDNSLookup {
	my $ipaddress=shift;
 	$NbOfDNSLookupAsked++;
	use Socket; $AFINET=AF_INET;
	my $tid=0;
	$tid=$MaxNbOfThread?eval(&quot;threads-&gt;self-&gt;tid()&quot;):0;
	if ($Debug) { debug(&quot;  ***** Thread id $tid: MakeDNSlookup started (for $ipaddress)&quot;,4); }
	my $lookupresult=gethostbyaddr(pack(&quot;C4&quot;,split(/\./,$ipaddress)),$AFINET);	# This is very slow, may took 20 seconds
	if (! $lookupresult || $lookupresult =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ || ! IsAscii($lookupresult)) {
		$TmpDNSLookup{$ipaddress}='*';
	}
	else {
		$TmpDNSLookup{$ipaddress}=$lookupresult;
	}
	if ($Debug) { debug(&quot;  ***** Thread id $tid: MakeDNSlookup done ($ipaddress resolved into $TmpDNSLookup{$ipaddress})&quot;,4); }
	delete $threadarray{$ipaddress};
	return;
}

#-----------------------------------------------------------------------------
# Function:     WriteRecordsReadyInQueue
# Input:        -
# Return:       0
#-----------------------------------------------------------------------------
sub WriteRecordsReadyInQueue {
	my $logfilechosen=shift;
	if ($Debug) { debug(&quot;Check head of queue to write records ready to flush (QueueCursor=$QueueCursor, QueueSize=&quot;.(scalar keys %QueueRecords).&quot;)&quot;,4); }
	while ( $QueueHostsToResolve{$QueueCursor} &amp;&amp; ( ($QueueHostsToResolve{$QueueCursor} eq '*') || ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}}) || ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}) ) ) {
		# $QueueCursor point to a ready record
		if ($QueueHostsToResolve{$QueueCursor} eq '*') {
			if ($Debug) { debug(&quot; First elem in queue is ready. No change on it. We pull it.&quot;,4); }
		}
		else {
			if ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}}) {
				if ($MyDNSTable{$QueueHostsToResolve{$QueueCursor}} ne '*') {
					$QueueRecords{$QueueCursor}=~s/$QueueHostsToResolve{$QueueCursor}/$MyDNSTable{$QueueHostsToResolve{$QueueCursor}}/;
					if ($Debug) { debug(&quot; First elem in queue has been resolved (found in MyDNSTable $MyDNSTable{$QueueHostsToResolve{$QueueCursor}}). We pull it.&quot;,4); }
				}
			}
			elsif ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}) {
				if ($TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}} ne '*') {
					$QueueRecords{$QueueCursor}=~s/$QueueHostsToResolve{$QueueCursor}/$TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}/;
					if ($Debug) { debug(&quot; First elem in queue has been resolved (found in TmpDNSLookup $TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}}). We pull it.&quot;,4); }
				}
			}
		}
		# Record is ready, we output it.
		if ($AddFileNum)  { print &quot;$logfilechosen &quot;; }
		if ($AddFileName) { print &quot;$LogFileToDo{$logfilechosen} &quot;; }
		# see if we need to dump fields
		if ($PrintFields &amp;&amp; $LastLogNum != $logfilechosen){
			print($Fields[$logfilechosen].&quot;\n&quot;);
			$LastLogNum = $logfilechosen;
		}
		print &quot;$QueueRecords{$QueueCursor}\n&quot;;
		delete $QueueRecords{$QueueCursor};
		delete $QueueHostsToResolve{$QueueCursor};
		$QueueCursor++;
	}
	return 0;
}

#-----------------------------------------------------------------------------
# Function:     Check if thread are enabled or not
# Input:        -
# Return:       -
#-----------------------------------------------------------------------------
sub Check_Thread_Use {
	if ($] &gt;= 5.008) {	for (0..@ARGV-1) { if ($ARGV[$_] =~ /^-dnslookup[:=](\d{1,2})/i) {
		if ($UseThread) {
			if (!eval ('require &quot;threads.pm&quot;;')) { &amp;error(&quot;Failed to load perl module 'threads' required for multi-threaded DNS lookup&quot;.($@?&quot;: $@&quot;:&quot;&quot;)); }
			if (!eval ('require &quot;threads/shared.pm&quot;;')) { &amp;error(&quot;Failed to load perl module 'threads::shared' required for multi-threaded DNS lookup&quot;.($@?&quot;: $@&quot;:&quot;&quot;)); }
		}
		else { &amp;error(&quot;Multi-thread is disabled in default version of this script.\nYou must manually edit the file '$0' to comment/uncomment all\nlines marked with 'ENABLETHREAD' string to enable multi-threading&quot;); }
		} }
	}
}


#-----------------------------------------------------------------------------
# MAIN
#-----------------------------------------------------------------------------
($DIR=$0) =~ s/([^\/\\]*)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1;

# Get parameters (Note: $MaxNbOfThread is already known
my $cpt=1;
for (0..@ARGV-1) {
	if ($ARGV[$_] =~ /^-/) {
		if ($ARGV[$_] =~ /debug=(\d)/i) { $Debug=$1; }
		elsif ($ARGV[$_] =~ /dnscache=/i) { $DNSLookup||=2; $DNSCache=$ARGV[$_]; $DNSCache =~ s/-dnscache=//; }
		elsif ($ARGV[$_] =~ /dnslookup[:=](\d{1,2})/i) { $DNSLookup||=1; $MaxNbOfThread=$1; }
		elsif ($ARGV[$_] =~ /dnslookup/i) { $DNSLookup||=1; }
		elsif ($ARGV[$_] =~ /showsteps/i) { $ShowSteps=1; }
		elsif ($ARGV[$_] =~ /addfilenum/i) { $AddFileNum=1; }
		elsif ($ARGV[$_] =~ /addfilename/i) { $AddFileName=1; }
		elsif ($ARGV[$_] =~ /stoponfirsteof/i) { $StopOnFirstEof=1; }
		elsif ($ARGV[$_] =~ /printfields/i) { $PrintFields=1; }
		elsif ($ARGV[$_] =~ /ignoremissing/i) { $IgnoreMissing=1; }
		else { print &quot;Unknown argument $ARGV[$_] ignored\n&quot;; }
	}
	elsif ($ARGV[$_] =~ /addfolder=(.*)$/i) {
   		@ParamFile = addDirectory($1, @ParamFile);
	}
	else {
		push @ParamFile, $ARGV[$_];
		$cpt++;
	}
}
if ($Debug) { $|=1; }

if ($Debug) {
	debug(ucfirst($PROG).&quot; - $VERSION - Perl $^X $]&quot;,1);
	debug(&quot;DNSLookup=$DNSLookup&quot;);
	debug(&quot;DNSCache=$DNSCache&quot;);
	debug(&quot;MaxNbOfThread=$MaxNbOfThread&quot;);
}

# Disallow MaxNbOfThread and Perl &lt; 5.8
if ($] &lt; 5.008 &amp;&amp; $MaxNbOfThread) {
	error(&quot;Multi-threaded DNS lookup is only supported with Perl 5.8 or higher (not $]). Use -dnslookup option instead&quot;);
}

# Warning, there is a memory hole in ActiveState perl version (in delete functions)
if ($^X =~ /activestate/i || $^X =~ /activeperl/i) {
	# TODO Add a warning

}

if (scalar @ParamFile == 0) {
	print &quot;----- $PROG $VERSION (c) Laurent Destailleur -----\n&quot;;
	print &quot;$PROG allows you to get one unique output log file, sorted on date,\n&quot;;
	print &quot;built from particular sources:\n&quot;;
	print &quot; - It can read several input log files,\n&quot;;
	print &quot; - It can read .gz/.bz2/.xz log files,\n&quot;;
	print &quot; - It can also makes a fast reverse DNS lookup to replace\n&quot;;
	print &quot;   all IP addresses into host names in resulting log file.\n&quot;;
	print &quot;$PROG comes with ABSOLUTELY NO WARRANTY. It's a free software\n&quot;;
	print &quot;distributed with a GNU General Public License (See COPYING.txt file).\n&quot;;
	print &quot;$PROG is part of AWStats but can be used alone as a log merger\n&quot;;
	print &quot;or resolver before using any other log analyzer.\n&quot;;
	print &quot;\n&quot;;
	print &quot;Usage:\n&quot;;
	print &quot;  $PROG.$Extension [options] file\n&quot;;
	print &quot;  $PROG.$Extension [options] file1 ... filen\n&quot;;
	print &quot;  $PROG.$Extension [options] *.*\n&quot;;
	print &quot;  $PROG.$Extension [options] addfolder=dirname\n&quot;;
	print &quot;  perl $PROG.$Extension [options] *.* &gt; newfile\n&quot;;
	print &quot;Options:\n&quot;;
	print &quot;  -dnslookup      make a reverse DNS lookup on IP adresses\n&quot;;
	print &quot;  -dnslookup=n    same with a n parallel threads instead of serial requests\n&quot;;
	print &quot;  -dnscache=file  make DNS lookup from cache file first before network lookup\n&quot;;
	print &quot;  -showsteps      print on stderr benchmark information every $NBOFLINESFORBENCHMARK lines\n&quot;;
	print &quot;  -addfilenum     if used with several files, file number can be added in first\n&quot;;
	print &quot;  -addfilename    if used with several files, file name can be added in first\n&quot;;
	print &quot;                  field of output file. This can be used to add a cluster id\n&quot;;
	print &quot;                  when log files come from several load balanced computers.\n&quot;;
	print &quot;  -stoponfirsteof Stop processing when any logfile reaches end-of-file.\n&quot;;
	print &quot;  -printfields    For IIS or W3C logs, prints the latest field header for\n&quot;;
	print &quot;                  the currentlog file when switching between log file entries\n&quot;;
	print &quot;                  so that the parsercan automatically determine which fields\n&quot;;
	print &quot;                  are avaiable.\n&quot;;
	print &quot;  -ignoremissing  will not fail if a log file is missing\n&quot;;
	print &quot;\n&quot;;
	
	print &quot;This runs $PROG in command line to open one or several\n&quot;;
	print &quot;server log files to merge them (sorted on date) and/or to make a reverse\n&quot;;
	print &quot;DNS lookup (if asked). The result log file is sent on standard output.\n&quot;;
	print &quot;Note: $PROG is not a 'sort' tool to sort one file. It's a\n&quot;;
	print &quot;software able to output sorted log records (with a reverse DNS lookup\n&quot;;
	print &quot;included or not) even if log records are dispatched in several files.\n&quot;;
	print &quot;Each of thoose files must be already independently sorted itself\n&quot;;
	print &quot;(but that is the case in all web server log files). So you can use it\n&quot;;
	print &quot;for load balanced log files or to group several old log files.\n&quot;;
	print &quot;\n&quot;;
	print &quot;Don't forget that the main goal of logresolvemerge is to send log records to\n&quot;;
	print &quot;a log analyzer in a sorted order without merging files on disk (NO NEED\n&quot;;
	print &quot;OF DISK SPACE AT ALL) and without loading files into memory (NO NEED\n&quot;;
	print &quot;OF MORE MEMORY). Choose of output records is done on the fly.\n&quot;;
	print &quot;\n&quot;;
	print &quot;So logresolvemerge is particularly useful when you want to output several\n&quot;;
	print &quot;and/or large log files in a fast process, with no use of disk or\n&quot;;
	print &quot;more memory, and in a chronological order through a pipe (to be used by a log\n&quot;;
	print &quot;analyzer).\n&quot;;
	print &quot;\n&quot;;
	print &quot;Note: If input records are not 'exactly' sorted but 'nearly' sorted (this\n&quot;;
	print &quot;occurs with heavy servers), this is not a problem, the output will also\n&quot;;
	print &quot;be 'nearly' sorted but a few log analyzers (like AWStats) knowns how to deal\n&quot;;
	print &quot;with such logs.\n&quot;;
	print &quot;\n&quot;;
	print &quot;WARNING: If log files are old MAC text files (lines ended with CR char), you\n&quot;;
	print &quot;can't run this tool on Win or Unix platforms.\n&quot;;
	print &quot;\n&quot;;
	print &quot;WARNING: Because of memory holes in ActiveState Perl version, use another\n&quot;;
	print &quot;Perl interpreter if you need to process large log files.\n&quot;;
	print &quot;\n&quot;;
	print &quot;Now supports/detects:\n&quot;;
	print &quot;  Automatic detection of log format\n&quot;;
	print &quot;  Files can be .gz/.bz2/.xz files if gzip/bzip2/xz tools are available in PATH.\n&quot;;
	print &quot;  Multithreaded reverse DNS lookup (several parallel requests) with Perl 5.8+.\n&quot;;
	print &quot;New versions and FAQ at http://www.awstats.org\n&quot;;
	exit 0;
}

# Get current time
my $nowtime=time;
my ($nowsec,$nowmin,$nowhour,$nowday,$nowmonth,$nowyear) = localtime($nowtime);
if ($nowyear &lt; 100) { $nowyear+=2000; } else { $nowyear+=1900; }
my $nowsmallyear=$nowyear;$nowsmallyear =~ s/^..//;
if (++$nowmonth &lt; 10) { $nowmonth = &quot;0$nowmonth&quot;; }
if ($nowday &lt; 10) { $nowday = &quot;0$nowday&quot;; }
if ($nowhour &lt; 10) { $nowhour = &quot;0$nowhour&quot;; }
if ($nowmin &lt; 10) { $nowmin = &quot;0$nowmin&quot;; }
if ($nowsec &lt; 10) { $nowsec = &quot;0$nowsec&quot;; }
# Get tomorrow time (will be used to discard some record with corrupted date (future date))
my ($tomorrowsec,$tomorrowmin,$tomorrowhour,$tomorrowday,$tomorrowmonth,$tomorrowyear) = localtime($nowtime+86400);
if ($tomorrowyear &lt; 100) { $tomorrowyear+=2000; } else { $tomorrowyear+=1900; }
my $tomorrowsmallyear=$tomorrowyear;$tomorrowsmallyear =~ s/^..//;
if (++$tomorrowmonth &lt; 10) { $tomorrowmonth = &quot;0$tomorrowmonth&quot;; }
if ($tomorrowday &lt; 10) { $tomorrowday = &quot;0$tomorrowday&quot;; }
if ($tomorrowhour &lt; 10) { $tomorrowhour = &quot;0$tomorrowhour&quot;; }
if ($tomorrowmin &lt; 10) { $tomorrowmin = &quot;0$tomorrowmin&quot;; }
if ($tomorrowsec &lt; 10) { $tomorrowsec = &quot;0$tomorrowsec&quot;; }
my $timetomorrow=$tomorrowyear.$tomorrowmonth.$tomorrowday.$tomorrowhour.$tomorrowmin.$tomorrowsec;	

# Init other parameters
$NBOFLINESFORBENCHMARK--;
if ($ENV{&quot;GATEWAY_INTERFACE&quot;}) { $DirCgi=''; }
if ($DirCgi &amp;&amp; !($DirCgi =~ /\/$/) &amp;&amp; !($DirCgi =~ /\\$/)) { $DirCgi .= '/'; }
if (! $DirData || $DirData eq '.') { $DirData=$DIR; }	# If not defined or chosen as &quot;.&quot; value then DirData is current dir
if (! $DirData)  { $DirData='.'; }						# If current dir not defined then we put it to &quot;.&quot;
$DirData =~ s/\/$//;

#my %monthlib =  ( &quot;01&quot;,&quot;$Message[60]&quot;,&quot;02&quot;,&quot;$Message[61]&quot;,&quot;03&quot;,&quot;$Message[62]&quot;,&quot;04&quot;,&quot;$Message[63]&quot;,&quot;05&quot;,&quot;$Message[64]&quot;,&quot;06&quot;,&quot;$Message[65]&quot;,&quot;07&quot;,&quot;$Message[66]&quot;,&quot;08&quot;,&quot;$Message[67]&quot;,&quot;09&quot;,&quot;$Message[68]&quot;,&quot;10&quot;,&quot;$Message[69]&quot;,&quot;11&quot;,&quot;$Message[70]&quot;,&quot;12&quot;,&quot;$Message[71]&quot; );
# monthnum must be in english because it's used to translate log date in apache log files which are always in english
my %monthnum =  ( &quot;Jan&quot;,&quot;01&quot;,&quot;jan&quot;,&quot;01&quot;,&quot;Feb&quot;,&quot;02&quot;,&quot;feb&quot;,&quot;02&quot;,&quot;Mar&quot;,&quot;03&quot;,&quot;mar&quot;,&quot;03&quot;,&quot;Apr&quot;,&quot;04&quot;,&quot;apr&quot;,&quot;04&quot;,&quot;May&quot;,&quot;05&quot;,&quot;may&quot;,&quot;05&quot;,&quot;Jun&quot;,&quot;06&quot;,&quot;jun&quot;,&quot;06&quot;,&quot;Jul&quot;,&quot;07&quot;,&quot;jul&quot;,&quot;07&quot;,&quot;Aug&quot;,&quot;08&quot;,&quot;aug&quot;,&quot;08&quot;,&quot;Sep&quot;,&quot;09&quot;,&quot;sep&quot;,&quot;09&quot;,&quot;Oct&quot;,&quot;10&quot;,&quot;oct&quot;,&quot;10&quot;,&quot;Nov&quot;,&quot;11&quot;,&quot;nov&quot;,&quot;11&quot;,&quot;Dec&quot;,&quot;12&quot;,&quot;dec&quot;,&quot;12&quot; );

if ($DNSCache) {
	if ($Debug) { debug(&quot;Load DNS Cache file $DNSCache&quot;,2); }
	open(my $CACHE, &quot;&lt;&quot;, &quot;$DNSCache&quot;) or error(&quot;Can't open cache file $DNSCache&quot;);
	while (&lt;$CACHE&gt;) {
		my ($time, $ip, $name) = split;
        if ($ip &amp;&amp; $name) {
            $name=&quot;$ip&quot; if $name eq '*';
    		$MyDNSTable{$ip}=$name;
        }
	}
	close $CACHE;
}

#-----------------------------------------------------------------------------
# PROCESSING CURRENT LOG(s)
#-----------------------------------------------------------------------------
my $NbOfLinesRead=0;
my $NbOfLinesParsed=0;
my $logfilechosen=0;
my $starttime=time();

# Define the LogFileToDo list
$cpt=1;
foreach my $key (0..(@ParamFile-1)) {
	if (($ParamFile[$key] !~ /\*/ &amp;&amp; $ParamFile[$key] !~ /\?/) || $ParamFile[$key] =~ /\|$/) {

		if ($Debug) { debug(&quot;DBG1 Log file $ParamFile[$key] is added to LogFileToDo with number $cpt.&quot;); }
		# Check for supported compression 
		if ($ParamFile[$key] =~ /$zcat_file/) {
			if ($Debug) { debug(&quot;GZIP compression detected for Log file $ParamFile[$key].&quot;); }
			# Modify the name to include the zcat command
			$ParamFile[$key] = $zcat . ' ' . $ParamFile[$key] . ' |';
		}
		elsif ($ParamFile[$key] =~ /$bzcat_file/) {
			if ($Debug) { debug(&quot;BZ2 compression detected for Log file $ParamFile[$key].&quot;); }
			# Modify the name to include the bzcat command
			$ParamFile[$key] = $bzcat . ' ' . $ParamFile[$key] . ' |';
		}
		elsif ($ParamFile[$key] =~ /$xzcat_file/) {
			if ($Debug) { debug(&quot;XZ compression detected for Log file $ParamFile[$key].&quot;); }
			# Modify the name to include the xzcat command
			$ParamFile[$key] = $xzcat . ' ' . $ParamFile[$key] . ' |';
		}

		$LogFileToDo{$cpt}=@ParamFile[$key];
		$cpt++;
		
	}
    else {
        my $DirFile=$ParamFile[$key]; $DirFile =~ s/([^\/\\]*)$//;
        $ParamFile[$key] = $1;
        if ($DirFile eq '') { $DirFile = '.'; }
        $ParamFile[$key] =~ s/\./\\\./g;
        $ParamFile[$key] =~ s/\*/\.\*/g;
        $ParamFile[$key] =~ s/\?/\./g;
        if ($Debug) { debug(&quot;Search for file \&quot;$ParamFile[$key]\&quot; into \&quot;$DirFile\&quot;&quot;); }
        opendir(my $DIR,&quot;$DirFile&quot;);
        my @filearray = sort readdir $DIR;
        close $DIR;
        foreach my $i (0..$#filearray) {
            if (&quot;$filearray[$i]&quot; =~ /^$ParamFile[$key]$/ &amp;&amp; &quot;$filearray[$i]&quot; ne &quot;.&quot; &amp;&amp; &quot;$filearray[$i]&quot; ne &quot;..&quot;) {

                if ($Debug) { debug(&quot;DBG2 Log file $filearray[$i] is added to LogFileToDo with number $cpt.&quot;); }
                # Check for supported compression
                if ($filearray[$i] =~ /$zcat_file/) {
                    if ($Debug) { debug(&quot;GZIP compression detected for Log file $filearray[$i].&quot;); }
                    # Modify the name to include the zcat command
                    $LogFileToDo{$cpt}=$zcat . ' ' . &quot;$DirFile/$filearray[$i]&quot; . ' |';
                }
                elsif ($filearray[$i] =~ /$bzcat_file/) {
                    if ($Debug) { debug(&quot;BZ2 compression detected for Log file $filearray[$i].&quot;); }
                    # Modify the name to include the bzcat command
                    $LogFileToDo{$cpt}=$bzcat . ' ' . &quot;$DirFile/$filearray[$i]&quot; . ' |';
                }
                elsif ($filearray[$i] =~ /$xzcat_file/) {
                    if ($Debug) { debug(&quot;XZ compression detected for Log file $filearray[$i].&quot;); }
                    # Modify the name to include the xzcat command
                    $LogFileToDo{$cpt}=$xzcat . ' ' . &quot;$DirFile/$filearray[$i]&quot; . ' |';
                }
                else {
                    $LogFileToDo{$cpt}=&quot;$DirFile/$filearray[$i]&quot;;
                }
                $cpt++;
                
            }
        }
    }
}

# If no files to process
if (scalar keys %LogFileToDo == 0) {
	error(&quot;No input log file found&quot;);
}

# Open all log files
my %LOG;
if ($Debug) { debug(&quot;Start of processing &quot;.(scalar keys %LogFileToDo).&quot; log file(s), $MaxNbOfThread threads max&quot;); }
foreach my $logfilenb (keys %LogFileToDo) {
	if ($Debug) { debug(&quot;Open log file number $logfilenb: \&quot;$LogFileToDo{$logfilenb}\&quot;&quot;); }
	if ($IgnoreMissing){
		if (!open($LOG{$logfilenb},&quot;&lt;&quot;, &quot;$LogFileToDo{$logfilenb}&quot;)){
			debug(&quot;Couldn't open log file \&quot;$LogFileToDo{$logfilenb}\&quot; : $!&quot;);
			delete $LogFileToDo{$logfilenb};
		}
	}else{
		open($LOG{$logfilenb}, &quot;&lt;&quot;, &quot;$LogFileToDo{$logfilenb}&quot;) || error(&quot;Couldn't open log file \&quot;$LogFileToDo{$logfilenb}\&quot; : $!&quot;);
	}
	binmode $LOG{$logfilenb};	# To avoid pb of corrupted text log files with binary chars.
}

$QueueCursor=1;
STOPONFIRSTEOF: while (1 == 1)
{
	# BEGIN Read new record
	# For each log file if logfilechosen is 0
	# If not, we go directly to log file instead of iterating over all keys for a match
	#----------------------------------------------------------------------------------
    my @readlist;
	if($logfilechosen == 0) {
	    @readlist = keys %LogFileToDo;
	} else {
	    @readlist = ($logfilechosen);
	}
	foreach my $logfilenb (@readlist)
	{
		if ($Debug) { debug(&quot;Search next record in file number $logfilenb&quot;,3); }
		# Read chosen log file until we found a record with good date or reaching end of file
		while (1 == 1) {
			my $LOG=$LOG{$logfilenb};
			$_=&lt;$LOG&gt;;	# Read new line
			if (! $_) 
			{							# No more records in log file number $logfilenb
				if ($Debug) { debug(&quot; No more records in file number $logfilenb&quot;,2); }
				delete $LogFileToDo{$logfilenb};
				if ($StopOnFirstEof) 
				{
					if ($Debug) { debug(&quot;Exiting loop due to EOF of logfile $logfilenb&quot;,1); }
					last STOPONFIRSTEOF;
				}
				last;
			}
		
			# Get the latest Fields header for printing IIS and W3C logs
			if ($PrintFields &amp;&amp; $_ =~ m/#Fields:/){
				my $field = $_;
				# strip whitespace
				$field =~ s/^\s+|\s+$//g;
				if (!$Fields[$logfilenb] || $field != $Fields[$logfilenb]){
					$Fields[$logfilenb] = $field;
					debug(&quot;Found new fields in $logfilenb: $Fields[$logfilenb]&quot;);
				}
			}
			
			$NbOfLinesRead++;
			chomp $_; s/\r$//;

			if (/^#/) { next; }									# Ignore comment lines (ISS writes such comments)
			if (/^!!/) { next; }								# Ignore comment lines (Webstar writes such comments)
			if (/^$/) { next; }									# Ignore blank lines (With ISS: happens sometimes, with Apache: possible when editing log file)

			$linerecord{$logfilenb}=$_; 

			# Check filters
			#----------------------------------------------------------------------

			# Split YYYY-MM-DD HH:MM:SS
			#    or DD/Month/YYYY:HH:MM:SS
			#    or MM/DD/YY\tHH:MM:SS
			#    or 9999.999
 			#    or Month DD HH:MM:SS
			my $year=0; my $month=0; my $day=0; my $hour=0; my $minute=0; my $second=0;
			if ($_ =~ /(\d\d\d\d)-(\d\d)-(\d\d)\s(\d\d):(\d\d):(\d\d)/) { $year=$1; $month=$2; $day=$3; $hour=$4; $minute=$5; $second=$6; }
			elsif ($_ =~ /\[(\d?\d)[\/:\s](\w+)[\/:\s](\d\d\d\d)[\/:\s](\d\d)[\/:\s](\d\d)[\/:\s](\d\d) /) { $year=$3; $month=$2; $day=$1; $hour=$4; $minute=$5; $second=$6; }
			elsif ($_ =~ /\w+ (\w+) {1,2}(\d?\d) (\d\d)[\/:\s](\d\d)[\/:\s](\d\d) (\d\d\d\d)/) { $year=$6; $month=$1; $day=$2; $hour=$3; $minute=$4; $second=$5; }
			elsif ($_ =~ /^(\d\d\d\d+\.\d\d\d) /)
			{
				my $timetime = strftime('%Y-%m-%d-%T', gmtime($1));
				$timetime =~ /(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d):(\d\d):(\d\d)/;
				$year=$1; $month=$2; $day=$3; $hour=$4; $minute=$5; $second=$6;
			}
 			elsif ($_ =~ /(\w+)\s\s?(\d?\d) (\d\d):(\d\d):(\d\d) /) {	# Month DD HH:MM:SS
 				$month=$1; $day=$2; $hour=$3; $minute=$4; $second=$5;
 				if (($monthnum{$month}&gt;$monthnum{$nowmonth}) || ($monthnum{$month}==$monthnum{$nowmonth} &amp;&amp;  $day&gt;$nowday)) {
 					$year=$nowyear-1;
 				}
                else { $year=$nowyear; }
 			}
			if (length $day == 1) { $day = &quot;0&quot;.$day; }

			if ($monthnum{$month}) { $month=$monthnum{$month}; }	# Change lib month in num month if necessary

			# Create $timerecord like YYYYMMDDHHMMSS
	 		$timerecord{$logfilenb}=int(&quot;$year$month$day$hour$minute$second&quot;);
			if ($timerecord{$logfilenb}&lt;10000000000000) {
				if ($Debug) { debug(&quot; This record is corrupted (no date found)&quot;,3); }
				$corrupted{$logfilenb}++;
				next;
			}
			if ($Debug) { debug(&quot; This is next record for file $logfilenb : timerecord=$timerecord{$logfilenb}&quot;,3); }
			
			# Sort and insert into timerecordorder, oldest at end/back of array
			# At the beginning, timerecordorder is empty. Then beceause the first pass is
			# a loop on each file to read each first line, the timerecordorder size is
			# number of input files.
			# After, each new loop, read only one new line, so timerecordorder size increase
			# by one but decrease just after by the pop command later.
			my $inserted=0;
			for(my $c=$#timerecordorder; $c&gt;=0 ; $c--) {
			    if($timerecord{$logfilenb} &lt;= $timerecord{$timerecordorder[$c]})
			    {
    				# Is older or equal than index at $c, add after
				    $timerecordorder[$c + 1]=$logfilenb;
				    $inserted = 1;
				    last;
			    } else {
				    $timerecordorder[$c + 1]=$timerecordorder[$c];
			    }
			}
			if(! $inserted) {
			    $timerecordorder[0] = $logfilenb;
			}

			last;
		}
	}
	# END Read new lines for each log file. After this, following var are filled
	# $timerecord{$logfilenb}
	# @timerecordorder array

	# We choose which record of which log file to process
	if ($Debug) { debug(&quot;Choose which record of which log file to process&quot;,3); }
	$logfilechosen=pop(@timerecordorder);
	if(!defined($logfilechosen)) { last; }              # No more record to process 
	
	# Record is chosen
	if ($Debug) { debug(&quot; We chose to qualify record of file number $logfilechosen&quot;,3); }
	if ($Debug) { debug(&quot;  Record is $linerecord{$logfilechosen}&quot;,3); }
			
	# Record is approved. We found a new line to parse in file number $logfilechosen
	#-------------------------------------------------------------------------------
	$NbOfLinesParsed++;
	if ($ShowSteps) {
		if ((++$NbOfLinesShowsteps &amp; $NBOFLINESFORBENCHMARK) == 0) {
			my $delay=(time()-$starttime)||1;
			print STDERR &quot;$NbOfLinesParsed lines processed (&quot;.(1000*$delay).&quot; ms, &quot;.int($NbOfLinesShowsteps/$delay).&quot; lines/seconds)\n&quot;;
		}
	}

	# Do DNS lookup
	#--------------------
	my $Host='';
	my $ip=0;
	if ($DNSLookup) {			# DNS lookup is 1 or 2
		if ($linerecord{$logfilechosen} =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/) { $ip=4; $Host=$1; }	# IPv4
		elsif ($linerecord{$logfilechosen} =~ /([0-9A-F]*:)/i) { $ip=6; $Host=$1; }						# IPv6
		if ($ip) {
			# Check in static DNS cache file
			if ($MyDNSTable{$Host}) {
				if ($Debug) { debug(&quot;  DNS lookup asked for $Host and found in static DNS cache file: $MyDNSTable{$Host}&quot;,4); }
			}
			elsif ($DNSLookup==1) {
				# Check in session cache (dynamic DNS cache file + session DNS cache)
				if (! $threadarray{$Host} &amp;&amp; ! $TmpDNSLookup{$Host}) {
					if (@SkipDNSLookupFor &amp;&amp; &amp;SkipDNSLookup($Host)) {
						$TmpDNSLookup{$Host}='*';
						if ($Debug) { debug(&quot;  No need of reverse DNS lookup for $Host, skipped at user request.&quot;,4); }
					}
					else {
						if ($ip == 4) {
							# Create or not a new thread
							if ($MaxNbOfThread) {
								if (! $threadarray{$Host}) {	# No thread already launched for $Host
									while ((scalar keys %threadarray) &gt;= $MaxNbOfThread) {
										if ($Debug) { debug(&quot; $MaxNbOfThread thread running reached, so we wait&quot;,4); }
										sleep 1;
									}
									$threadarray{$Host}=1;		# Semaphore to tell thread for $Host is active
#									my $t = new Thread \&amp;MakeDNSLookup, $Host;
									my $t = threads-&gt;create(sub { MakeDNSLookup($Host) });
									if (! $t) { error(&quot;Failed to create new thread&quot;); }
									if ($Debug) { debug(&quot; Reverse DNS lookup for $Host queued in thread &quot;.$t-&gt;tid,4); }
									$t-&gt;detach();	# We don't need to keep return code
								}
								else {
									if ($Debug) { debug(&quot; Reverse DNS lookup for $Host already queued in a thread&quot;); }
								}
								# Here, this is the only way, $TmpDNSLookup{$Host} can be not defined
							} else {
								&amp;MakeDNSLookup($Host);
								if ($Debug) { debug(&quot;  Reverse DNS lookup for $Host done: $TmpDNSLookup{$Host}&quot;,4); }
							}				
						}
						elsif ($ip == 6) {
							$TmpDNSLookup{$Host}='*';
							if ($Debug) { debug(&quot;  Reverse DNS lookup for $Host not available for IPv6&quot;,4); }
						}
					}
				} else {
					if ($Debug) { debug(&quot;  Reverse DNS lookup already queued or done for $Host: $TmpDNSLookup{$Host}&quot;,4); }
				}
			}
			else {
				if ($Debug) { debug(&quot;  DNS lookup by static DNS cache file asked for $Host but not found.&quot;,4); }
			}
		}
		else {
			if ($Debug) { debug(&quot;  DNS lookup asked for $Host but this is not an IP address.&quot;,4); }
			$DNSLookupAlreadyDone=$LogFileToDo{$logfilechosen};
		}
	}
	else {
		if ($linerecord{$logfilechosen} =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/) { $ip=4; $Host=$1; }	# IPv4
		elsif ($linerecord{$logfilechosen} =~ /([0-9A-F]*:)/i) { $ip=6; $Host=$1; }						# IPv6
		if ($Debug) { debug(&quot;  No DNS lookup asked.&quot;,4); }
	}

	# Put record in record queue
	if ($Debug) { debug(&quot;Add record $NbOfLinesParsed in record queue (with host to resolve = &quot;.($Host?$Host:'*').&quot;)&quot;,4); }
	$QueueRecords{$NbOfLinesParsed}=$linerecord{$logfilechosen};

	# Put record in host queue
	# If there is a host to resolve, we add line to queue with value of host to resolve
	# $Host is '' (no ip found) or is ip
	if ($DNSLookup==0) {
		$QueueHostsToResolve{$NbOfLinesParsed}='*';
	}
	if ($DNSLookup==1) { 
		$QueueHostsToResolve{$NbOfLinesParsed}=$Host?$Host:'*';
	}
	if ($DNSLookup==2) {
		$QueueHostsToResolve{$NbOfLinesParsed}=$MyDNSTable{$Host}?$Host:'*';
	}

	# Print all records in head of queue that are ready
	&amp;WriteRecordsReadyInQueue($logfilechosen);
	
}	# End of processing new record. Loop on next one.

if ($Debug) { debug(&quot;End of processing log file(s)&quot;); }

# Close all log files
foreach my $logfilenb (keys %LogFileToDo) {
	if ($Debug) { debug(&quot;Close log file number $logfilenb&quot;); }
	close($LOG{$logfilenb}) || error(&quot;Command for pipe '$LogFileToDo{$logfilenb}' failed&quot;);
}

while ( $QueueHostsToResolve{$QueueCursor} &amp;&amp; $QueueHostsToResolve{$QueueCursor} ne '*' &amp;&amp; ! $MyDNSTable{$QueueHostsToResolve{$QueueCursor}} &amp;&amp; ! $TmpDNSLookup{$QueueHostsToResolve{$QueueCursor}} ) {
	sleep 1;
	# Print all records in head of queue that are ready
	&amp;WriteRecordsReadyInQueue($logfilechosen);
}

# Waiting queue is empty
if ($MaxNbOfThread) {
	foreach my $t (threads-&gt;list()) {
		if ($Debug) { debug(&quot;Join thread $t&quot;); }
		$t-&gt;join();
	}
}

# DNSLookup warning
if ($DNSLookup==1 &amp;&amp; $DNSLookupAlreadyDone) {
	warning(&quot;Warning: $PROG has detected that some host names were already resolved in your logfile $DNSLookupAlreadyDone.\nIf DNS lookup was already made by the logger (web server) in ALL your log files, you should not use -dnslookup option to increase $PROG speed.&quot;);
}

if ($Debug) {
	debug(&quot;Total nb of read lines: $NbOfLinesRead&quot;);
	debug(&quot;Total nb of parsed lines: $NbOfLinesParsed&quot;);
	debug(&quot;Total nb of DNS lookup asked: $NbOfDNSLookupAsked&quot;);
}

#if ($DNSCache) {
#	open(CACHE, &quot;&gt;$DNSCache&quot;) or die;
#	foreach (keys %TmpDNSLookup) {
#		$TmpDNSLookup{$_}=&quot;*&quot; if $TmpDNSLookup{$_} eq &quot;ip&quot;;
#		print CACHE &quot;0\t$_\t$TmpDNSLookup{$_}\n&quot;;
#	}
#	close CACHE;
#}

0;	# Do not remove this line
</textarea><br><input class='btn' type='submit' value='Save Changes'></form><a class='btn' href='?dir=%2Fusr%2Flocal%2Fcpanel%2F3rdparty%2Fbin'>Back to Directory</a>