Enduro/X

0001 #!/usr/bin/perl
0002 ##
0003 ## @brief @(#) Enduro/X log analyzer - join all the log files in the chronological order
0004 ##
0005 ## @file uvlog
0006 ##
0007 ## -----------------------------------------------------------------------------
0008 ## Enduro/X Middleware Platform for Distributed Transaction Processing
0009 ## Copyright (C) 2009-2016, ATR Baltic, Ltd. All Rights Reserved.
0010 ## Copyright (C) 2017-2023, Mavimax, Ltd. All Rights Reserved.
0011 ## This software is released under one of the following licenses:
0012 ## AGPL (with Java and Go exceptions) or Mavimax's license for commercial use.
0013 ## See LICENSE file for full text.
0014 ## -----------------------------------------------------------------------------
0015 ## AGPL license:
0016 ##
0017 ## This program is free software; you can redistribute it and/or modify it under
0018 ## the terms of the GNU Affero General Public License, version 3 as published
0019 ## by the Free Software Foundation;
0020 ##
0021 ## This program is distributed in the hope that it will be useful, but WITHOUT ANY
0022 ## WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
0023 ## PARTICULAR PURPOSE. See the GNU Affero General Public License, version 3
0024 ## for more details.
0025 ##
0026 ## You should have received a copy of the GNU Affero General Public License along
0027 ## with this program; if not, write to the Free Software Foundation, Inc.,
0028 ## 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0029 ##
0030 ## -----------------------------------------------------------------------------
0031 ## A commercial use license is available from Mavimax, Ltd
0032 ## contact@mavimax.com
0033 ## -----------------------------------------------------------------------------
0034 ##
0035
0036 use Time::Local;
0037
0038 # The output format of the log could be like:
0039 #sss.ffffff|last 8 chars of file-name|<the log line>
0040 #sss.ffffff|last 8 chars of file-name|<the log line continues... (i.e. repeate of header, but dump data)>
0041 # sss.ffffff -> time diff from pervious line
0042 ################################################################################
0043 # Config
0044 ################################################################################
0045
0046 #
0047 # Mask used for line detection date/time value extraction
0048 #
0049 my $M_line_mask = '^.*:.*:[0-6]:.*:[ 0-9]*:[0-9a-fA-F]*:[0-9]{3}:([0-9]{8}):([0-9]{12}):';
0050 my $M_line_front = '^(.*:.*:[0-6]:.*:[ 0-9]*:[0-9a-fA-F]*:[0-9]{3}:[0-9]{8}:[0-9]{12}):';
0051
0052 my $M_last_tstamp = -1; # Last timestamp of printed line, to calculate timediff.
0053
0054 my $M_time_diff_cur = 0; # current time diff
0055
0056 my $M_debug = 0; # add internal debugs
0057 ################################################################################
0058 # Globals
0059 ################################################################################
0060 my @M_files;
0061 my @M_files_eof;
0062 my @M_lines_buffered;
0063 my @M_lines_enqueued;
0064
0065 ##
0066 # Read one line from file
0067 # @param idx File index
0068 # @return the line read (can include newlines).
0069 ##
0070 sub read_line {
0071     my $idx = shift;
0072     my $done = 0;
0073
0074     # Take the current line from buffer
0075     $M_lines_enqueued[$idx] = $M_lines_buffered[$idx];
0076
0077     # Clear the buffer
0078     $M_lines_buffered[$idx] = "";
0079
0080     # Read stuff into @M_lines_enqueued
0081
0082     # Gives something like GLOB(0x23d6a08) if not using my $fh..
0083     my $fh = $M_files[$idx];
0084     while (!$M_files_eof[$idx] && !$done)
0085     {
0086         print STDERR "About to read line...\n" unless (!$M_debug);
0087         my $line = <$fh>;
0088         print STDERR "Read..\n" unless (!$M_debug);
0089
0090         if (defined $line)
0091         {
0092             print STDERR "Defined..\n" unless (!$M_debug);
0093             chomp $line;
0094             print STDERR "Got line: [$line] - ".length($M_lines_enqueued[$idx])."\n" unless (!$M_debug);
0095
0096             if ($line =~ /$M_line_mask/)
0097             {
0098                 print STDERR "Line matched pattern...\n" unless (!$M_debug);
0099                 if (length($M_lines_enqueued[$idx]))
0100                 {
0101                     # buffer up the line...
0102                     $M_lines_buffered[$idx] = $line;
0103                     # We are done... but the incomplete lines might be still there
0104                     $done = 1;
0105                 }
0106                 else
0107                 {
0108                     $M_lines_enqueued[$idx] = $line;
0109                 }
0110             }
0111             # So if we have original record.
0112             elsif (length($M_lines_enqueued[$idx]))
0113             {
0114                 print STDERR "Adding aux line...\n" unless (!$M_debug);
0115                 $M_lines_enqueued[$idx] = $M_lines_enqueued[$idx]."\n".$line;
0116             }
0117         }
0118         else
0119         {
0120             print STDERR "Not defined.. $idx\n" unless (!$M_debug);
0121             $M_files_eof[$idx] = 1;
0122         }
0123     }
0124     print STDERR "read_line done..\n" unless (!$M_debug);
0125 }
0126
0127 ##
0128 # Check is any line enqueued
0129 ##
0130 sub is_lines_enqueued {
0131
0132     for ($i=0; $i<(scalar @M_lines_enqueued); $i++)
0133     {
0134         #if any line...
0135         if (length(@M_lines_enqueued[$i]))
0136         {
0137             return 1;
0138         }
0139     }
0140
0141     return 0;
0142 }
0143
0144 ##
0145 # Ensure that all readable lines are enqueued..
0146 ##
0147 sub enqueue_lines {
0148
0149     for ($i=0; $i<(scalar @M_files); $i++)
0150     {
0151         print STDERR "enqueue_lines: $i\n" unless (!$M_debug);
0152         my $fh = $M_files[$i];
0153
0154         print STDERR "enqueue_lines: $i 2 len=".length($M_lines_enqueued[$i])." eof:".$M_files_eof[$i]." file: ".$ARGV[$i]."\n" unless (!$M_debug);
0155
0156         if (0==length($M_lines_enqueued[$i]) && !$M_files_eof[$i])
0157         {
0158             print STDERR "enqueue_lines: $i - reading line...\n" unless (!$M_debug);
0159             # Read the line...
0160             read_line($i);
0161
0162             print STDERR "enqueue_lines: $i - line read ok\n" unless (!$M_debug);
0163         }
0164     }
0165     print STDERR "enqueue_lines: return\n" unless (!$M_debug);
0166 }
0167
0168 #
0169 # Get the line tstamp...
0170 #
0171 sub get_line_tstamp {
0172     my $idx = shift;
0173
0174     my ($date, $time) = ($M_lines_enqueued[$idx] =~/$M_line_mask/);
0175     my ($year, $month, $day) = ($date=~/([0-9]{4})([0-9]{2})([0-9]{2})/);
0176     my ($hour, $minute, $second, $ffffff) = ($time =~/^([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{6})/);
0177
0178
0179     print STDERR "date=$date time=$time year=$year month=$month day=$day\n" unless (!$M_debug);
0180     print STDERR "hour=$hour minute=$minute second=$second ffffff=$ffffff\n" unless (!$M_debug);
0181
0182     my $epoch = timelocal($second,$minute,$hour,$day,$month-1,$year);
0183
0184     $epoch = $epoch* 1000000 + $ffffff;
0185
0186     print STDERR "Extracted epoch: [$epoch] for the $idx\n" unless (!$M_debug);
0187
0188     return $epoch;
0189 }
0190
0191 ##
0192 # Return oldest line from the array..
0193 ##
0194 sub get_oldest_line {
0195
0196     my $old_idx = -1;
0197     my $old_time = 99999999999999999999999;
0198
0199
0200     print STDERR "Into get_oldest_line()\n" unless (!$M_debug);
0201
0202     for ($i=0; $i<(scalar @M_lines_enqueued); $i++)
0203     {
0204
0205         print STDERR "Into get_oldest_line() -> $i\n" unless (!$M_debug);
0206
0207         # Extract time EPOCH
0208         if (length($M_lines_enqueued[$i]))
0209         {
0210             my $tmp = get_line_tstamp($i);
0211
0212             if ($tmp < $old_time)
0213             {
0214                 $old_idx = $i;
0215                 $old_time = $tmp;
0216             }
0217         }
0218     }
0219
0220     if ($old_idx > -1)
0221     {
0222         if ($M_last_tstamp > 0)
0223         {
0224             $M_time_diff_cur = $old_time - $M_last_tstamp;
0225             print STDERR "Timediff = $M_time_diff_cur old_time=$old_time\n" unless (!$M_debug);
0226         }
0227         $M_last_tstamp = $old_time;
0228         return $old_idx;
0229     }
0230     else
0231     {
0232         $M_time_diff_cur = 0;
0233         $M_last_tstamp = 0;
0234
0235         return -1;
0236     }
0237 }
0238
0239 ##
0240 # Print the line by index
0241 ##
0242 sub print_line  {
0243
0244     my $idx = shift;
0245
0246     my ($line_front) = ($time =~ /\Q$M_line_front/);
0247
0248     my $lead = sprintf("%03d.%06d|%-14.14s",
0249         ($M_time_diff_cur / 1000000), ($M_time_diff_cur % 1000000), substr($ARGV[$idx], -14));
0250
0251     $line_front = $lead.'|'.$line_front;
0252
0253     my $first = 1;
0254     foreach (split(/\n/, $M_lines_enqueued[$idx]))
0255     {
0256         if ($first)
0257         {
0258                 print("$lead|$_\n");
0259         }
0260         else
0261         {
0262                 print("$line_front|$_\n");
0263         }
0264     }
0265
0266     $M_lines_enqueued[$idx] = "";
0267 }
0268
0269 ################################################################################
0270 # Open handles
0271 ################################################################################
0272
0273 print STDERR "DEBUG: Done open handles\n" unless (!$M_debug);
0274
0275 my $idx = 0;
0276 for my $i (0 .. $#ARGV)
0277 {
0278     my $file = $ARGV[$i];
0279     print STDERR "Got name: [$file]\n" unless (!$M_debug);
0280
0281     open $M_files[$i], '<', $file or die "Error opening file $file: $!";
0282     #push @M_files, $fh;
0283
0284     $M_lines_enqueued[$i] = "";
0285     $M_lines_buffered[$i] = "";
0286     $M_files_eof[$i] = 0;
0287 }
0288
0289 print STDERR "DEBUG: Done open handles\n" unless (!$M_debug);
0290
0291 ################################################################################
0292 # Main loop
0293 ################################################################################
0294
0295 # Enqueue lines
0296 print STDERR "DEBUG: About to enqueue lines...\n" unless (!$M_debug);
0297 enqueue_lines();
0298
0299 do
0300 {
0301     # Get the oldest line
0302     print STDERR "DEBUG: Get oldes line...\n" unless (!$M_debug);
0303     my $idx = get_oldest_line();
0304
0305     print STDERR "DEBUG: Got oldest line at $idx -> $M_lines_enqueued[$idx]\n" unless (!$M_debug);
0306
0307     # This should print the filename, timestamp, and calculate the time diff
0308     # from previous line.
0309     if ($idx>-1)
0310     {
0311         print_line($idx);
0312     }
0313
0314     # Enqueue lines
0315     print STDERR "DEBUG: About to enqueue lines...\n" unless (!$M_debug);
0316     enqueue_lines();
0317     print STDERR "DEBUG: enqueued_lines\n" unless (!$M_debug);
0318
0319 } while (is_lines_enqueued());
0320
0321
0322 ################################################################################
0323 # Close handles
0324 ################################################################################
0325 foreach my $handle (@M_files) {
0326     close $handle;
0327 }
0328 # vim: set ts=4 sw=4 et smartindent: