more on this

1 files changed, 18 insertions, 72 deletions

diff --git a/foostats.pl b/foostats.pl
index 6fd9634..9cefc8c 100644
--- a/foostats.pl
+++ b/foostats.pl
@@ -18,8 +18,7 @@ no warnings qw(experimental::refaliasing);
 # TODO: Blog post about this script and the new Perl features used.
 # TODO NEXT:
 # 1) Implement replicator
-# 2) Also merge the results
-# 3) Write out a nice output from each merged file
+# 2) Write out a nice output from each merged file, also merge if multiple hosts results
 
 package Foostats::Logreader {
   use Digest::SHA3 'sha3_512_base64';
@@ -340,11 +339,11 @@ package Foostats::Replicator {
   use File::Basename;
   use Time::Piece;
   use LWP::UserAgent;
-  use Scalar::Util qw(looks_like_number);
+  use String::Util qw(endswith);
   
   sub new ($class, %args) { bless \%args, $class }
 
-  sub replicate_and_merge ($self, $partner_node) {
+  sub replicate ($self, $partner_node) {
     say "Replicating from $partner_node";
 
     for my $proto (qw(gemini web)) {
@@ -357,16 +356,8 @@ package Foostats::Replicator {
         $self->replicate_file(
           "https://$partner_node/foostats/$dest_file", 
           $self->{stats_dir} . '/' . $dest_file,
-          $count < 3, # Always replicate the newest 3 files.
+          $count++ < 3, # Always replicate the newest 3 files.
         );
-
-        $self->merge_files(
-          $self->{stats_dir} . "/${file_base}.*.json.gz",
-          $self->{stats_dir} . "/merged.${file_base}.json.gz",
-          $count < 3, # Always merge the newest 3 files.
-        );
-
-        $count++;
       }
     }
   }
@@ -376,7 +367,7 @@ package Foostats::Replicator {
     return if !$force && -f $dest_file;
     return # UNDO
 
-    print "Replicating $remote_url to $dest_file (force:$force)... ";
+    say "Replicating $remote_url to $dest_file (force:$force)... ";
     my $response = LWP::UserAgent->new->get($remote_url);
     unless ($response->is_success) {
       say "\nFailed to fetch the file: " . $response->status_line;
@@ -391,24 +382,7 @@ package Foostats::Replicator {
     say 'done';
   }
 
-  sub merge_files ($self, $file_glob, $dest_file, $force) {
-    # $dest_file already exists, not replicating it
-    return if !$force && -f $dest_file;
-
-    say "Merging $file_glob to $dest_file (force:$force)... ";
-    my %jsons;
-    $jsons{$_} = _read_json_gz($_) for glob $file_glob;
-    my $merged_json = _merge_jsons(\%jsons);
-
-    say "Writing $dest_file";
-    open my $fd, '>:gzip', "$dest_file.tmp" or die "$dest_file.tmp: $!";
-    $merged_json = encode_json $merged_json;
-    print $fd $merged_json;
-    close $fd;
-    rename "$dest_file.tmp", $dest_file or die "$dest_file.tmp: $!";
-  }
-
-  sub _read_json_gz ($file_path) {
+   sub _read_json_gz ($file_path) {
       # TODO: Refactor to JSON helper package
       say "Reading $file_path";
       open my $fd, '<:gzip', $file_path or die "$file_path: $!";
@@ -417,34 +391,6 @@ package Foostats::Replicator {
       return $json;
   }
 
-  sub _merge_jsons ($jsons) {
-    my %result;
-
-    my sub merge_scalars ($a, $b) {
-      die "Unable to merge $a and $b as they don't look like numbers"
-        unless looks_like_number($a) and looks_like_number($b);
-      return $a + $b;
-    }
-
-    my sub deep ($a, $b) {
-      my ($t, $t2) = (ref $a, ref $b);
-      die "Can't merge types $t and $t2" if $t ne $t2;
-
-      if ($t eq 'HASH') {
-        while (my ($key, $val ) = each %$b) {
-          if (!exists $a->{$key}) {
-            $a->{$key} = $val;
-          } elsif ($t eq 'SCALAR') {
-            $a->{$key} = merge_scalars $a->{$key}, $val;
-          }
-        }
-      }
-    }
-
-    deep \%result, $_ for values %$jsons;
-    return \%result;
-  }
-
   sub _last_month_dates () {
     my $today = localtime;
     my @last_week;
@@ -473,27 +419,27 @@ package main {
     $out->write;
   }
 
-  sub replicate_and_merge ($stats_dir, $partner_node) {
-    Foostats::Replicator->new(stats_dir => $stats_dir)->replicate_and_merge($partner_node);
+  sub replicate ($stats_dir, $partner_node) {
+    Foostats::Replicator->new(stats_dir => $stats_dir)->replicate($partner_node);
   }
 
-  sub pretty_print () { say 'pretty_print not yet implemented' }
+  sub report () { say 'report not yet implemented' }
 
-  my ($parse_logs, $replicate_and_merge, $pretty_print, $all);
+  my ($parse_logs, $replicate, $report, $all);
 
   # With default values
   my $stats_dir = '/var/www/htdocs/buetow.org/self/foostats';
   my $partner_node = hostname eq 'fishfinger.buetow.org' 
                    ? 'blowfish.buetow.org' : 'fishfinger.buetow.org';
 
-  GetOptions 'parse-logs'          => \$parse_logs,
-             'replicate_and_merge' => \$replicate_and_merge,
-             'pretty-print'        => \$pretty_print,
-             'all'                 => \$all,
-             'stats-dir'           => \$stats_dir,
-             'partner-node'        => \$partner_node;
+  GetOptions 'parse-logs'   => \$parse_logs,
+             'replicate'    => \$replicate,
+             'pretty-print' => \$report,
+             'all'          => \$all,
+             'stats-dir'    => \$stats_dir,
+             'partner-node' => \$partner_node;
 
   parse_logs $stats_dir if $parse_logs or $all;
-  replicate_and_merge $stats_dir, $partner_node if $replicate_and_merge or $all;
-  pretty_print $stats_dir if $pretty_print or $all;
+  replicate $stats_dir, $partner_node if $replicate or $all;
+  report $stats_dir if $report or $all;
 }