diff options
| -rw-r--r-- | frontends/scripts/foostats.pl | 518 |
1 files changed, 280 insertions, 238 deletions
diff --git a/frontends/scripts/foostats.pl b/frontends/scripts/foostats.pl index e6ef1c5..eca3997 100644 --- a/frontends/scripts/foostats.pl +++ b/frontends/scripts/foostats.pl @@ -17,13 +17,6 @@ no warnings qw(experimental::refaliasing); use constant VERSION => 'v0.1.0'; -# TODO: Blog post about this script and the new Perl features used. -# TODO NEXT: -# * Write out a nice output from each merged file, also merge if multiple hosts results -# * Fix bug with .gmi.*.gmi in the log parser -# * Nicely formatted .txt output by stats by count by date -# * Print out all UAs, to add new excludes/blocked IPs - # Package: FileHelper — small file/JSON helpers # - Purpose: Atomic writes, gzip JSON read/write, and line reading. # - Notes: Dies on I/O errors; JSON encoding uses core JSON. @@ -103,7 +96,7 @@ package DateHelper { my $today = localtime; my @dates; - for my $days_ago (0 .. 30) { + for my $days_ago (1 .. 31) { my $date = $today - ($days_ago * 24 * 60 * 60); push @dates, @@ -113,7 +106,6 @@ package DateHelper { return @dates; } - } # Package: Foostats::Logreader — parse and normalize logs @@ -131,10 +123,11 @@ package Foostats::Logreader { # - Purpose: Glob for gemini-related logs; env override for testing. # - Return: glob pattern string. sub gemini_logs_glob { $ENV{FOOSTATS_GEMINI_LOGS_GLOB} // '/var/log/daemon*' } + # Sub: web_logs_glob # - Purpose: Glob for web access logs; env override for testing. # - Return: glob pattern string. - sub web_logs_glob { $ENV{FOOSTATS_WEB_LOGS_GLOB} // '/var/www/logs/access.log*' } + sub web_logs_glob { $ENV{FOOSTATS_WEB_LOGS_GLOB} // '/var/www/logs/access.log*' } # Sub: anonymize_ip # - Purpose: Classify IPv4/IPv6 and map IP to a stable SHA3-512 base64 hash. @@ -324,7 +317,6 @@ package Foostats::Logreader { } } -# TODO: Write filter summary at the end of the filter log. # Package: Foostats::Filter — request filtering and logging # - Purpose: Identify odd URI patterns and excessive requests per second per IP. # - Notes: Maintains an in-process blocklist for the current run. @@ -670,6 +662,7 @@ package Foostats::Replicator { # Package: Foostats::Merger — merge per-host daily stats into a single view # - Purpose: Merge multiple node files per day into totals and unique counts. package Foostats::Merger { + # Removed Data::Dumper (debug-only) per review. # Sub: merge # - Purpose: Produce merged stats for the last month (date => stats hashref). @@ -806,8 +799,7 @@ package Foostats::Merger { $merge{$key}, $_->{page_ips}->{$key}, sub ($key) { - $key =~ s/\.html$/.../; - $key =~ s/\.gmi$/.../; + $key =~ s/\.gmi$/\.html/; $key; } ) for @stats; @@ -947,104 +939,115 @@ package Foostats::Reporter { # - Params: $content (str) Gemtext. # - Return: HTML string (fragment). sub gemtext_to_html { - my ($content) = @_; - my $html = ""; - my $in_code_block = 0; - my $in_list = 0; - my @lines = split /\n/, $content; - my @code_block_lines = (); - - for my $line (@lines) { + my ($content) = @_; + my $html = ""; + my @lines = split /\n/, $content; + my $i = 0; + + while ($i < @lines) { + my $line = $lines[$i]; + if ($line =~ /^```/) { - if ($in_code_block) { - - # End code block - check if it's a table - if (is_ascii_table(\@code_block_lines)) { - $html .= convert_ascii_table_to_html(\@code_block_lines); - } - else { - $html .= "<pre>\n"; - for my $code_line (@code_block_lines) { - $html .= encode_entities($code_line) . "\n"; - } - $html .= "</pre>\n"; - } - @code_block_lines = (); - $in_code_block = 0; + my @block_lines; + $i++; # Move past the opening ``` + while ($i < @lines && $lines[$i] !~ /^```/) { + push @block_lines, $lines[$i]; + $i++; } - else { - $in_code_block = 1; - } - next; + $html .= _gemtext_to_html_code_block(\@block_lines); } - - if ($in_code_block) { - push @code_block_lines, $line; - next; + elsif ($line =~ /^### /) { + $html .= _gemtext_to_html_heading($line); } - - - - # Check if we need to close a list - if ($in_list && $line !~ /^\* /) { - $html .= "</ul>\n"; - $in_list = 0; + elsif ($line =~ /^## /) { + $html .= _gemtext_to_html_heading($line); } - - # Headers - if ($line =~ /^### (.*)/) { - $html .= "<h3>" . encode_entities($1) . "</h3>\n"; - } - elsif ($line =~ /^## (.*)/) { - $html .= "<h2>" . encode_entities($1) . "</h2>\n"; + elsif ($line =~ /^# /) { + $html .= _gemtext_to_html_heading($line); } - elsif ($line =~ /^# (.*)/) { - $html .= "<h1>" . encode_entities($1) . "</h1>\n"; + elsif ($line =~ /^=> /) { + $html .= _gemtext_to_html_link($line); } - - # Links - elsif ($line =~ /^=> (\S+)\s+(.*)/) { - my ($url, $text) = ($1, $2); - - # Drop 365-day summary links from HTML output - if ($url =~ /(?:^|[\/.])365day_summary_\d{8}\.gmi$/) { - next; + elsif ($line =~ /^\* /) { + my @list_items; + while ($i < @lines && $lines[$i] =~ /^\* /) { + push @list_items, $lines[$i]; + $i++; } - - # Convert .gmi links to .html - $url =~ s/\.gmi$/\.html/; - $html .= "<p><a href=\"" . encode_entities($url) . "\">" . encode_entities($text) . "</a></p>\n"; + $html .= _gemtext_to_html_list(\@list_items); + $i--; # Decrement to re-evaluate the current line in the outer loop } - - # Bullet points - elsif ($line =~ /^\* (.*)/) { - if (!$in_list) { - $html .= "<ul>\n"; - $in_list = 1; - } - $html .= "<li>" . linkify_text($1) . "</li>\n"; + elsif ($line !~ /^\s*$/) { + $html .= _gemtext_to_html_paragraph($line); } + # Else, it's a blank line, which we skip for compact output. + $i++; + } - # Empty line - skip to avoid excessive spacing - elsif ($line =~ /^\s*$/) { + return $html; + } - # Skip empty lines for more compact output + sub _gemtext_to_html_code_block { + my ($lines) = @_; + if (is_ascii_table($lines)) { + return convert_ascii_table_to_html($lines); + } + else { + my $html = "<pre>\n"; + for my $code_line (@$lines) { + $html .= encode_entities($code_line) . "\n"; } + $html .= "</pre>\n"; + return $html; + } + } - # Regular text - else { - $html .= "<p>" . linkify_text($line) . "</p>\n"; - } + sub _gemtext_to_html_heading { + my ($line) = @_; + if ($line =~ /^### (.*)/) { + return "<h3>" . encode_entities($1) . "</h3>\n"; + } + elsif ($line =~ /^## (.*)/) { + return "<h2>" . encode_entities($1) . "</h2>\n"; } + elsif ($line =~ /^# (.*)/) { + return "<h1>" . encode_entities($1) . "</h1>\n"; + } + return ''; + } - # Close list if still open - if ($in_list) { - $html .= "</ul>\n"; + sub _gemtext_to_html_link { + my ($line) = @_; + if ($line =~ /^=> (\S+)\s+(.*)/) { + my ($url, $text) = ($1, $2); + + # Drop 365-day summary links from HTML output + return '' if $url =~ /(?:^|[\/.])365day_summary_\d{8}\.gmi$/; + + # Convert .gmi links to .html + $url =~ s/\.gmi$/\.html/; + return "<p><a href=\"" . encode_entities($url) . "\">" . encode_entities($text) . "</a></p>\n"; } + return ''; + } + sub _gemtext_to_html_list { + my ($lines) = @_; + my $html = "<ul>\n"; + for my $line (@$lines) { + if ($line =~ /^\* (.*)/) { + $html .= "<li>" . linkify_text($1) . "</li>\n"; + } + } + $html .= "</ul>\n"; return $html; } + sub _gemtext_to_html_paragraph { + my ($line) = @_; + return "<p>" . linkify_text($line) . "</p>\n"; + } + # Check if the lines form an ASCII table # Sub: is_ascii_table # - Purpose: Heuristically detect if a code block is an ASCII table. @@ -1133,7 +1136,7 @@ package Foostats::Reporter { my ($host, $path) = ($1, $2 // ''); my $has_ellipsis = index($t, '...') != -1 || index(($path // ''), '...') != -1; my $is_gemini = defined($path) && $path =~ /\.gmi(?:[?#].*)?$/i; - my $scheme = $is_gemini ? 'gemini' : 'https'; + my $scheme = 'https'; # If truncated, fall back to host root my $href = @@ -1171,7 +1174,8 @@ package Foostats::Reporter { my $href = _guess_href($core); if ($href) { - $out .= sprintf('<a href="%s.html">%s</a>%s', + $href =~ s/\.gmi$/\.html/i; + $out .= sprintf('<a href="%s">%s</a>%s', encode_entities($href), encode_entities($core), encode_entities($trail)); } else { @@ -1260,129 +1264,157 @@ $content }; } - # Sub: report - # - Purpose: Generate daily .gmi and .html reports per date, then summaries and index. - # - Params: $stats_dir, $output_dir, $html_output_dir, %merged (date => stats). - # - Return: undef. - sub report { - my ($stats_dir, $output_dir, $html_output_dir, %merged) = @_; - for my $date (sort { $b cmp $a } keys %merged) { - my $stats = $merged{$date}; - next unless $stats->{count}; - - my ($year, $month, $day) = $date =~ /(\d{4})(\d{2})(\d{2})/; + # Sub: should_generate_daily_report + # - Purpose: Check if a daily report should be generated based on file existence and age. + # - Params: $date (YYYYMMDD), $report_path (str), $html_report_path (str). + # - Return: 1 if report should be generated, 0 otherwise. + sub should_generate_daily_report { + my ($date, $report_path, $html_report_path) = @_; - # Check if .gmi file exists and its age based on date in filename - my $report_path = "$output_dir/$date.gmi"; - my $html_report_path = "$output_dir/$date.html"; + my ($year, $month, $day) = $date =~ /(\d{4})(\d{2})(\d{2})/; - # Calculate age of the data based on date in filename - my $today = Time::Piece->new(); - my $file_date = Time::Piece->strptime($date, '%Y%m%d'); - my $age_days = ($today - $file_date) / (24 * 60 * 60); + # Calculate age of the data based on date in filename + my $today = Time::Piece->new(); + my $file_date = Time::Piece->strptime($date, '%Y%m%d'); + my $age_days = ($today - $file_date) / (24 * 60 * 60); - if (-e $report_path && -e $html_report_path) { + if (-e $report_path && -e $html_report_path) { - # Files exist - if ($age_days <= 3) { + # Files exist + if ($age_days <= 3) { - # Data is recent (within 3 days), regenerate it - say "Regenerating daily report for $year-$month-$day (data age: " - . sprintf("%.1f", $age_days) - . " days)"; - } - else { - # Data is old (older than 3 days), skip if files exist - say "Skipping daily report for $year-$month-$day (files exist, data age: " - . sprintf("%.1f", $age_days) - . " days)"; - next; - } + # Data is recent (within 3 days), regenerate it + say "Regenerating daily report for $year-$month-$day (data age: " + . sprintf("%.1f", $age_days) + . " days)"; + return 1; } else { - # File doesn't exist, generate it - say "Generating new daily report for $year-$month-$day (file doesn't exist, data age: " + # Data is old (older than 3 days), skip if files exist + say "Skipping daily report for $year-$month-$day (files exist, data age: " . sprintf("%.1f", $age_days) . " days)"; + return 0; } + } + else { + # File doesn't exist, generate it + say "Generating new daily report for $year-$month-$day (file doesn't exist, data age: " + . sprintf("%.1f", $age_days) + . " days)"; + return 1; + } + } - my $report_content = ""; - - $report_content .= "## Stats for $year-$month-$day\n\n"; - - # Feed counts first - $report_content .= "### Feed Statistics\n\n"; - my @feed_rows; - push @feed_rows, [ 'Total', $stats->{feed_ips}{'Total'} // 0 ]; - push @feed_rows, [ 'Gemini Gemfeed', $stats->{feed_ips}{'Gemini Gemfeed'} // 0 ]; - push @feed_rows, [ 'Gemini Atom', $stats->{feed_ips}{'Gemini Atom'} // 0 ]; - push @feed_rows, [ 'Web Gemfeed', $stats->{feed_ips}{'Web Gemfeed'} // 0 ]; - push @feed_rows, [ 'Web Atom', $stats->{feed_ips}{'Web Atom'} // 0 ]; - $report_content .= "```\n"; - $report_content .= format_table([ 'Feed Type', 'Count' ], \@feed_rows); - $report_content .= "\n```\n\n"; - - # Top 50 URLs next - $report_content .= "### Top 50 URLs\n\n"; - my @url_rows; - my $urls = $stats->{page_ips}{urls}; - my @sorted_urls = - sort { ($urls->{$b} // 0) <=> ($urls->{$a} // 0) } - keys %$urls; - my $truncated = @sorted_urls > 50; - @sorted_urls = @sorted_urls[ 0 .. 49 ] if $truncated; - - for my $url (@sorted_urls) { - push @url_rows, [ $url, $urls->{$url} // 0 ]; - } + sub generate_feed_stats_section { + my ($stats) = @_; + my $report_content = "### Feed Statistics\n\n"; + my @feed_rows; + push @feed_rows, [ 'Total', $stats->{feed_ips}{'Total'} // 0 ]; + push @feed_rows, [ 'Gemini Gemfeed', $stats->{feed_ips}{'Gemini Gemfeed'} // 0 ]; + push @feed_rows, [ 'Gemini Atom', $stats->{feed_ips}{'Gemini Atom'} // 0 ]; + push @feed_rows, [ 'Web Gemfeed', $stats->{feed_ips}{'Web Gemfeed'} // 0 ]; + push @feed_rows, [ 'Web Atom', $stats->{feed_ips}{'Web Atom'} // 0 ]; + $report_content .= "```\n"; + $report_content .= format_table([ 'Feed Type', 'Count' ], \@feed_rows); + $report_content .= "\n```\n\n"; + return $report_content; + } - # Truncate URLs to fit within 100-character rows - truncate_urls_for_table(\@url_rows, 'Unique Visitors'); - $report_content .= "```\n"; - $report_content .= format_table([ 'URL', 'Unique Visitors' ], \@url_rows); - $report_content .= "\n```\n"; - if ($truncated) { - $report_content .= "\n... and more (truncated to 50 entries).\n"; - } - $report_content .= "\n"; + sub generate_top_n_table { + my (%args) = @_; + my $title = $args{title}; + my $data = $args{data}; + my $headers = $args{headers}; + my $limit = $args{limit} // 50; + my $is_url = $args{is_url} // 0; + + my $report_content = "### $title\n\n"; + my @rows; + my @sorted_keys = + sort { ($data->{$b} // 0) <=> ($data->{$a} // 0) } + keys %$data; + my $truncated = @sorted_keys > $limit; + @sorted_keys = @sorted_keys[ 0 .. $limit - 1 ] if $truncated; + + for my $key (@sorted_keys) { + push @rows, [ $key, $data->{$key} // 0 ]; + } - # Other tables afterwards: Hosts, then Summary - $report_content .= "### Page Statistics (by Host)\n\n"; - my @host_rows; - my $hosts = $stats->{page_ips}{hosts}; - my @sorted_hosts = - sort { ($hosts->{$b} // 0) <=> ($hosts->{$a} // 0) } - keys %$hosts; + if ($is_url) { + truncate_urls_for_table(\@rows, $headers->[1]); + } - $truncated = @sorted_hosts > 50; - @sorted_hosts = @sorted_hosts[ 0 .. 49 ] if $truncated; + $report_content .= "```\n"; + $report_content .= format_table($headers, \@rows); + $report_content .= "\n```\n"; + if ($truncated) { + $report_content .= "\n... and more (truncated to $limit entries).\n"; + } + $report_content .= "\n"; + return $report_content; + } - for my $host (@sorted_hosts) { - push @host_rows, [ $host, $hosts->{$host} // 0 ]; - } - $report_content .= "```\n"; - $report_content .= format_table([ 'Host', 'Unique Visitors' ], \@host_rows); - $report_content .= "\n```\n"; - if ($truncated) { - $report_content .= "\n... and more (truncated to 50 entries).\n"; - } - $report_content .= "\n"; - - # Summary last - $report_content .= "### Summary\n\n"; - my $total_requests = - ($stats->{count}{gemini} // 0) + ($stats->{count}{web} // 0); - $report_content .= "* Total requests: $total_requests\n"; - $report_content .= - "* Filtered requests: " . ($stats->{count}{filtered} // 0) . "\n"; - $report_content .= - "* Gemini requests: " . ($stats->{count}{gemini} // 0) . "\n"; - $report_content .= - "* Web requests: " . ($stats->{count}{web} // 0) . "\n"; - $report_content .= - "* IPv4 requests: " . ($stats->{count}{IPv4} // 0) . "\n"; - $report_content .= - "* IPv6 requests: " . ($stats->{count}{IPv6} // 0) . "\n\n"; + sub generate_top_urls_section { + my ($stats) = @_; + return generate_top_n_table( + title => 'Top 50 URLs', + data => $stats->{page_ips}{urls}, + headers => [ 'URL', 'Unique Visitors' ], + is_url => 1, + ); + } + + sub generate_top_hosts_section { + my ($stats) = @_; + return generate_top_n_table( + title => 'Page Statistics (by Host)', + data => $stats->{page_ips}{hosts}, + headers => [ 'Host', 'Unique Visitors' ], + ); + } + + sub generate_summary_section { + my ($stats) = @_; + my $report_content = "### Summary\n\n"; + my $total_requests = + ($stats->{count}{gemini} // 0) + ($stats->{count}{web} // 0); + $report_content .= "* Total requests: $total_requests\n"; + $report_content .= + "* Filtered requests: " . ($stats->{count}{filtered} // 0) . "\n"; + $report_content .= + "* Gemini requests: " . ($stats->{count}{gemini} // 0) . "\n"; + $report_content .= + "* Web requests: " . ($stats->{count}{web} // 0) . "\n"; + $report_content .= + "* IPv4 requests: " . ($stats->{count}{IPv4} // 0) . "\n"; + $report_content .= + "* IPv6 requests: " . ($stats->{count}{IPv6} // 0) . "\n\n"; + return $report_content; + } + + # Sub: report + # - Purpose: Generate daily .gmi and .html reports per date, then summaries and index. + # - Params: $stats_dir, $output_dir, $html_output_dir, %merged (date => stats). + # - Return: undef. + sub report { + my ($stats_dir, $output_dir, $html_output_dir, %merged) = @_; + for my $date (sort { $b cmp $a } keys %merged) { + my $stats = $merged{$date}; + next unless $stats->{count}; + + my ($year, $month, $day) = $date =~ /(\d{4})(\d{2})(\d{2})/; + + my $report_path = "$output_dir/$date.gmi"; + my $html_report_path = "$html_output_dir/$date.html"; + + next unless should_generate_daily_report($date, $report_path, $html_report_path); + + my $report_content = "## Stats for $year-$month-$day\n\n"; + $report_content .= generate_feed_stats_section($stats); + $report_content .= generate_top_urls_section($stats); + $report_content .= generate_top_hosts_section($stats); + $report_content .= generate_summary_section($stats); # Add links to summary reports (only monthly) $report_content .= "## Related Reports\n\n"; @@ -1433,6 +1465,7 @@ $content # Order: feed counts -> Top URLs -> daily top 3 for last 30 days -> other tables $report_content .= build_feed_statistics_section(\@dates, \%merged); + $report_content .= build_feed_statistics_daily_average_section(\@dates, \%merged); # Aggregate and add top lists my ($all_hosts, $all_urls) = aggregate_hosts_and_urls(\@dates, \%merged); @@ -1465,6 +1498,37 @@ $content } } + sub build_feed_statistics_daily_average_section { + my ($dates, $merged) = @_; + + my %totals; + my $days_with_stats = 0; + + for my $date (@$dates) { + my $stats = $merged->{$date}; + next unless $stats->{feed_ips}; + $days_with_stats++; + + for my $key (keys %{ $stats->{feed_ips} }) { + $totals{$key} += $stats->{feed_ips}{$key}; + } + } + + return "" unless $days_with_stats > 0; + + my @avg_rows; + for my $key (sort keys %totals) { + my $avg = sprintf("%.2f", $totals{$key} / $days_with_stats); + push @avg_rows, [ $key, $avg ]; + } + + my $content = "### Feed Statistics Daily Average (Last 30 Days)\n\n```\n"; + $content .= format_table([ 'Feed Type', 'Daily Average' ], \@avg_rows); + $content .= "\n```\n\n"; + + return $content; + } + # Sub: build_report_header # - Purpose: Header section for summary reports. # - Params: $today (Time::Piece), $days (int default 30). @@ -1596,29 +1660,15 @@ $content return (\%all_hosts, \%all_urls); } - # Sub: build_top_hosts_section - # - Purpose: Build Top-50 hosts table for the aggregated period. - # - Params: $all_hosts (hashref), $days (int default 30). - # - Return: gemtext string. sub build_top_hosts_section { my ($all_hosts, $days) = @_; $days //= 30; - my $content = "## Top 50 Hosts (${days}-Day Total)\n\n```\n"; - - my @host_rows; - my @sorted_hosts = - sort { $all_hosts->{$b} <=> $all_hosts->{$a} } keys %$all_hosts; - @sorted_hosts = @sorted_hosts[ 0 .. 49 ] if @sorted_hosts > 50; - - for my $host (@sorted_hosts) { - push @host_rows, [ $host, $all_hosts->{$host} ]; - } - - $content .= format_table([ 'Host', 'Visitors' ], \@host_rows); - $content .= "\n```\n\n"; - - return $content; + return generate_top_n_table( + title => "Top 50 Hosts (${days}-Day Total)", + data => $all_hosts, + headers => [ 'Host', 'Visitors' ], + ); } # Sub: build_top_urls_section @@ -1629,24 +1679,12 @@ $content my ($all_urls, $days) = @_; $days //= 30; - my $content = "## Top 50 URLs (${days}-Day Total)\n\n```\n"; - - my @url_rows; - my @sorted_urls = - sort { $all_urls->{$b} <=> $all_urls->{$a} } keys %$all_urls; - @sorted_urls = @sorted_urls[ 0 .. 49 ] if @sorted_urls > 50; - - for my $url (@sorted_urls) { - push @url_rows, [ $url, $all_urls->{$url} ]; - } - - # Truncate URLs to fit within 100-character rows - truncate_urls_for_table(\@url_rows, 'Visitors'); - - $content .= format_table([ 'URL', 'Visitors' ], \@url_rows); - $content .= "\n```\n\n"; - - return $content; + return generate_top_n_table( + title => "Top 50 URLs (${days}-Day Total)", + data => $all_urls, + headers => [ 'URL', 'Visitors' ], + is_url => 1, + ); } # Sub: build_summary_links @@ -1700,7 +1738,10 @@ $content @sorted = @sorted[ 0 .. $limit - 1 ]; my @rows; - for my $u (@sorted) { push @rows, [ $u, $urls->{$u} // 0 ]; } + for my $u (@sorted) { + $u =~ s/\.gmi$/\.html/; + push @rows, [ $u, $urls->{$u} // 0 ]; + } truncate_urls_for_table(\@rows, 'Visitors'); $content .= "```\n" . format_table([ 'URL', 'Visitors' ], \@rows) . "\n```\n\n"; } @@ -1772,6 +1813,7 @@ $content } package main; + # Package: main — CLI entrypoint and orchestration # - Purpose: Parse options and invoke parse/replicate/report flows. use Getopt::Long; |