diff options
| -rw-r--r-- | foostats.pl | 90 |
1 files changed, 18 insertions, 72 deletions
diff --git a/foostats.pl b/foostats.pl index 6fd9634..9cefc8c 100644 --- a/foostats.pl +++ b/foostats.pl @@ -18,8 +18,7 @@ no warnings qw(experimental::refaliasing); # TODO: Blog post about this script and the new Perl features used. # TODO NEXT: # 1) Implement replicator -# 2) Also merge the results -# 3) Write out a nice output from each merged file +# 2) Write out a nice output from each merged file, also merge if multiple hosts results package Foostats::Logreader { use Digest::SHA3 'sha3_512_base64'; @@ -340,11 +339,11 @@ package Foostats::Replicator { use File::Basename; use Time::Piece; use LWP::UserAgent; - use Scalar::Util qw(looks_like_number); + use String::Util qw(endswith); sub new ($class, %args) { bless \%args, $class } - sub replicate_and_merge ($self, $partner_node) { + sub replicate ($self, $partner_node) { say "Replicating from $partner_node"; for my $proto (qw(gemini web)) { @@ -357,16 +356,8 @@ package Foostats::Replicator { $self->replicate_file( "https://$partner_node/foostats/$dest_file", $self->{stats_dir} . '/' . $dest_file, - $count < 3, # Always replicate the newest 3 files. + $count++ < 3, # Always replicate the newest 3 files. ); - - $self->merge_files( - $self->{stats_dir} . "/${file_base}.*.json.gz", - $self->{stats_dir} . "/merged.${file_base}.json.gz", - $count < 3, # Always merge the newest 3 files. - ); - - $count++; } } } @@ -376,7 +367,7 @@ package Foostats::Replicator { return if !$force && -f $dest_file; return # UNDO - print "Replicating $remote_url to $dest_file (force:$force)... "; + say "Replicating $remote_url to $dest_file (force:$force)... "; my $response = LWP::UserAgent->new->get($remote_url); unless ($response->is_success) { say "\nFailed to fetch the file: " . $response->status_line; @@ -391,24 +382,7 @@ package Foostats::Replicator { say 'done'; } - sub merge_files ($self, $file_glob, $dest_file, $force) { - # $dest_file already exists, not replicating it - return if !$force && -f $dest_file; - - say "Merging $file_glob to $dest_file (force:$force)... "; - my %jsons; - $jsons{$_} = _read_json_gz($_) for glob $file_glob; - my $merged_json = _merge_jsons(\%jsons); - - say "Writing $dest_file"; - open my $fd, '>:gzip', "$dest_file.tmp" or die "$dest_file.tmp: $!"; - $merged_json = encode_json $merged_json; - print $fd $merged_json; - close $fd; - rename "$dest_file.tmp", $dest_file or die "$dest_file.tmp: $!"; - } - - sub _read_json_gz ($file_path) { + sub _read_json_gz ($file_path) { # TODO: Refactor to JSON helper package say "Reading $file_path"; open my $fd, '<:gzip', $file_path or die "$file_path: $!"; @@ -417,34 +391,6 @@ package Foostats::Replicator { return $json; } - sub _merge_jsons ($jsons) { - my %result; - - my sub merge_scalars ($a, $b) { - die "Unable to merge $a and $b as they don't look like numbers" - unless looks_like_number($a) and looks_like_number($b); - return $a + $b; - } - - my sub deep ($a, $b) { - my ($t, $t2) = (ref $a, ref $b); - die "Can't merge types $t and $t2" if $t ne $t2; - - if ($t eq 'HASH') { - while (my ($key, $val ) = each %$b) { - if (!exists $a->{$key}) { - $a->{$key} = $val; - } elsif ($t eq 'SCALAR') { - $a->{$key} = merge_scalars $a->{$key}, $val; - } - } - } - } - - deep \%result, $_ for values %$jsons; - return \%result; - } - sub _last_month_dates () { my $today = localtime; my @last_week; @@ -473,27 +419,27 @@ package main { $out->write; } - sub replicate_and_merge ($stats_dir, $partner_node) { - Foostats::Replicator->new(stats_dir => $stats_dir)->replicate_and_merge($partner_node); + sub replicate ($stats_dir, $partner_node) { + Foostats::Replicator->new(stats_dir => $stats_dir)->replicate($partner_node); } - sub pretty_print () { say 'pretty_print not yet implemented' } + sub report () { say 'report not yet implemented' } - my ($parse_logs, $replicate_and_merge, $pretty_print, $all); + my ($parse_logs, $replicate, $report, $all); # With default values my $stats_dir = '/var/www/htdocs/buetow.org/self/foostats'; my $partner_node = hostname eq 'fishfinger.buetow.org' ? 'blowfish.buetow.org' : 'fishfinger.buetow.org'; - GetOptions 'parse-logs' => \$parse_logs, - 'replicate_and_merge' => \$replicate_and_merge, - 'pretty-print' => \$pretty_print, - 'all' => \$all, - 'stats-dir' => \$stats_dir, - 'partner-node' => \$partner_node; + GetOptions 'parse-logs' => \$parse_logs, + 'replicate' => \$replicate, + 'pretty-print' => \$report, + 'all' => \$all, + 'stats-dir' => \$stats_dir, + 'partner-node' => \$partner_node; parse_logs $stats_dir if $parse_logs or $all; - replicate_and_merge $stats_dir, $partner_node if $replicate_and_merge or $all; - pretty_print $stats_dir if $pretty_print or $all; + replicate $stats_dir, $partner_node if $replicate or $all; + report $stats_dir if $report or $all; } |