move

24 files changed, 1083 insertions, 0 deletions

diff --git a/frontends/Rexfile b/frontends/Rexfile
new file mode 100644
index 0000000..a58cc1d
--- /dev/null
+++ b/frontends/Rexfile
@@ -0,0 +1,286 @@
+# How to use:
+#
+# rex commons nsd_master nsd_slaves
+#
+# Why use Rex to automate my servers? Because Rex is KISS, Puppet, SALT and Chef
+# are not. So, why not use Ansible then? To use Ansible correctly you should also
+# install Python on the target machines (not mandatory, though. But better).
+# Rex is programmed in Perl and there is already Perl in the base system of OpenBSD.
+# Also, I find Perl > Python (my personal opinion).
+
+use Rex -feature => ['1.4'];
+use Rex::Logger;
+use File::Slurp;
+
+# REX CONFIG SECTION
+
+group frontends => 'blowfish.buetow.org', 'twofish.buetow.org';
+group dnsmaster => 'blowfish.buetow.org';
+group dnsslaves => 'twofish.buetow.org';
+
+user 'rex';
+sudo TRUE;
+
+parallelism 5;
+
+# CUSTOM (PERL-ish) CONFIG SECTION (what Rex can't do by itself)
+
+# Gather IPv6 addresses based on hostname.
+our $ipv6address = sub {
+  my $hostname = shift;
+  return '2a01:4f8:c17:20f1::42' if $hostname eq 'blowfish';
+  return '2401:c080:1000:45af:5400:3ff:fec6:ca1d' if $hostname eq 'twofish';
+  Rex::Logger::info("Unable to determine IPv6 address for $hostname", 'error');
+  return '::1';
+};
+
+# Bootstrapping the FQDN based on the server IP as the hostname and domain
+# facts aren't set yet due to the myname file in the first place.
+our $fqdns = sub {
+  my $ipv4 = shift;
+  return 'blowfish.buetow.org' if $ipv4 eq '23.88.35.144'; 
+  return 'twofish.buetow.org' if $ipv4 eq '108.160.134.135';
+  Rex::Logger::info("Unable to determine hostname for $ipv4", 'error');
+  return 'HOSTNAME-UNKNOWN.buetow.org';
+};
+
+our @dns_zones = qw/buetow.org dtail.dev foo.surf foo.zone irregular.ninja sidewalk.ninja snonux.de snonux.me/;
+
+sub secret {
+  my $secret = shift;
+  read_file($ENV{HOME} . '/.rexsecrets/' . $secret);
+}
+
+# UTILITY TASKS
+
+task 'id', group => 'frontends', sub { say run 'id' };
+task 'dump_info', group => 'frontends', sub { dump_system_information };
+
+# OPENBSD TASKS SECTION
+
+desc 'Install base stuff';
+task 'base', group => 'frontends',
+  sub {
+    pkg 'rsync', ensure => present;
+    pkg 'sudo', ensure => present;
+    pkg 'tig', ensure => present;
+    pkg 'vger', ensure => present;
+    pkg 'zsh', ensure => present;
+
+    append_if_no_such_line '/etc/rc.conf.local', 'pkg_scripts="uptimed httpd"';
+
+    file '/etc/myname',
+      content => template('./etc/myname.tpl', fqdns => $fqdns),
+      owner => 'root',
+      group => 'wheel',
+      mode => '644';
+  };
+
+desc 'Setup uptimed';
+task 'uptimed', group => 'frontends',
+  sub {
+    Rex::Logger::info('Setting up uptimed');
+    pkg 'uptimed', ensure => present;
+    service 'uptimed', ensure => 'started';
+  };
+
+desc 'Setup httpd';
+task 'httpd', group => 'frontends',
+  sub {
+    append_if_no_such_line '/etc/rc.conf.local', 'httpd_flags=';
+    #delete_lines_according_to qr{httpd_flags}, '/etc/rc.conf.local';
+
+    file '/etc/httpd.conf',
+      source => './etc/httpd.conf',
+      owner => 'root',
+      group => 'wheel',
+      mode => '644',
+      on_change => sub {
+        service 'httpd' => 'restart';
+      };
+    service 'httpd', ensure => 'started';
+  };
+
+desc 'Setup inetd';
+task 'inetd', group => 'frontends',
+  sub {
+    append_if_no_such_line '/etc/rc.conf.local', 'inetd_flags=';
+
+    file '/etc/inetd.conf',
+      source => './etc/inetd.conf',
+      owner => 'root',
+      group => 'wheel',
+      mode => '644',
+      on_change => sub {
+        service 'inetd' => 'restart';
+      };
+    service 'inetd', ensure => 'started';
+  };
+
+desc 'Setup relayd';
+task 'relayd', group => 'frontends',
+  sub {
+    append_if_no_such_line '/etc/rc.conf.local', 'relayd_flags=';
+
+    file '/etc/relayd.conf',
+      content => template('./etc/relayd.conf.tpl', ipv6address => $ipv6address),
+      owner => 'root',
+      group => 'wheel',
+      mode => '600',
+      on_change => sub {
+        service 'relayd' => 'restart';
+      };
+    service 'relayd', ensure => 'started';
+  };
+
+desc 'Setup OpenSMTPD';
+task 'smtpd', group => 'frontends',
+  sub {
+    file '/etc/mail/aliases',
+      source => './etc/mail/aliases',
+      owner => 'root',
+      group => 'wheel',
+      mode => '644',
+      on_change => sub {
+        say run 'newaliases';
+      };
+
+    file '/etc/mail/virtualdomains',
+      source => './etc/mail/virtualdomains',
+      owner => 'root',
+      group => 'wheel',
+      mode => '644',
+      on_change => sub {
+        service 'smtpd' => 'restart';
+      };
+
+    file '/etc/mail/virtualusers',
+      source => './etc/mail/virtualusers',
+      owner => 'root',
+      group => 'wheel',
+      mode => '644',
+      on_change => sub {
+        service 'smtpd' => 'restart';
+      };
+
+    file '/etc/mail/smtpd.conf',
+      content => template('./etc/mail/smtpd.conf.tpl', mail_hostname => sub {
+          my $hostname = shift;
+          return 'buetow.org' if $hostname eq 'blowfish';
+          return 'www.buetow.org' if $hostname eq 'twofish';
+          return 'buetow.org';
+        }),
+      owner => 'root',
+      group => 'wheel',
+      mode => '644',
+      on_change => sub {
+        service 'smtpd' => 'restart';
+      };
+
+    service 'smtpd', ensure => 'started';
+  };
+
+desc 'Setup DNS server';
+task 'nsd_master', group => 'dnsmaster',
+  sub {
+    my $restart = FALSE;
+    append_if_no_such_line '/etc/rc.conf.local', 'nsd_flags=';
+
+    file '/var/nsd/etc/key.conf',
+      content => template('./var/nsd/etc/key.conf.tpl',
+          nsd_secret => secret('nsd_secret')),
+      owner => 'root',
+      group => '_nsd',
+      mode => '640',
+      on_change => sub {
+        $restart = TRUE;
+      };
+
+    file '/var/nsd/etc/nsd.conf',
+      content => template('./var/nsd/etc/nsd.conf.master.tpl',
+          dns_zones => \@dns_zones),
+      owner => 'root',
+      group => '_nsd',
+      mode => '640',
+      on_change => sub {
+        $restart = TRUE;
+      };
+
+    for my $zone (@dns_zones) {
+      file "/var/nsd/zones/master/$zone.zone",
+        content => template("./var/nsd/zones/master/$zone.zone.tpl"),
+        owner => 'root',
+        group => 'wheel',
+        mode => '644',
+        on_change => sub {
+          $restart = TRUE;
+        };
+     }
+
+    service 'nsd' => 'restart' if $restart;
+    service 'nsd', ensure => 'started';
+  };
+
+desc 'Setup DNS slaves';
+task 'nsd_slaves', group => 'dnsslaves',
+  sub {
+    my $restart = FALSE;
+
+    file '/var/nsd/etc/key.conf',
+      content => template('./var/nsd/etc/key.conf.tpl',
+          nsd_secret => secret('nsd_secret')),
+      owner => 'root',
+      group => '_nsd',
+      mode => '640',
+      on_change => sub {
+        $restart = TRUE;
+      };
+
+    file '/var/nsd/etc/nsd.conf',
+      content => template('./var/nsd/etc/nsd.conf.slave.tpl',
+          dns_zones => \@dns_zones),
+      owner => 'root',
+      group => '_nsd',
+      mode => '640',
+      on_change => sub {
+        $restart = TRUE;
+      };
+
+    service 'nsd' => 'restart' if $restart;
+    service 'nsd', ensure => 'started';
+  };
+
+desc 'Setup HA';
+task 'ha', group => 'frontends',
+  sub {
+    file '/usr/local/bin/ha.pl',
+      source => './usr/local/bin/ha.pl',
+      owner => 'root',
+      group => 'wheel',
+      mode => '755';
+
+    file '/var/run/ha.status',
+      content => '# Initial HA status file',
+      owner => 'www',
+      group => 'wheel',
+      mode => '644',
+      no_overwrite => TRUE;
+  };
+
+# COMBINED TASKS SECTION
+
+desc 'Common configs of all hosts';
+task 'commons', group => 'frontends',
+  sub {
+    base();
+    uptimed();
+    httpd();
+    inetd();
+    relayd();
+    smtpd();
+    ha();
+  };
+
+1;
+
+# vim: syntax=perl
diff --git a/frontends/etc/httpd.conf b/frontends/etc/httpd.conf
new file mode 100644
index 0000000..41cde32
--- /dev/null
+++ b/frontends/etc/httpd.conf
@@ -0,0 +1,158 @@
+server "foo.zone" {
+  listen on * port 80
+  block return 302 "https://foo.zone"
+}
+
+server "www.foo.zone" {
+  listen on * port 80
+  block return 302 "https://www.foo.zone"
+}
+
+server "foo.zone" {
+  alias "www.foo.zone"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/foo.zone.fullchain.pem"
+    key "/etc/ssl/private/foo.zone.key"
+  }
+  location "/*" {
+    root "/htdocs/gemtexter/foo.zone"
+    directory auto index
+  }
+}
+
+server "irregular.ninja" {
+  listen on * port 80
+  block return 302 "https://irregular.ninja"
+}
+
+server "www.irregular.ninja" {
+  listen on * port 80
+  block return 302 "https://www.irregular.ninja"
+}
+
+server "irregular.ninja" {
+  alias "www.irregular.ninja"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/irregular.ninja.fullchain.pem"
+    key "/etc/ssl/private/irregular.ninja.key"
+  }
+  location "/*" {
+    root "/htdocs/irregular.ninja"
+    directory auto index
+  }
+}
+
+server "snonux.de" {
+  alias "www.snonux.de"
+  listen on * port 80
+        block return 302 "https://foo.zone$REQUEST_URI"
+}
+
+server "snonux.de" {
+  alias "www.snonux.de"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/snonux.de.fullchain.pem"
+    key "/etc/ssl/private/snonux.de.key"
+  }
+  block return 302 "https://foo.zone$REQUEST_URI"
+}
+
+server "foo.surf" {
+  alias "www.foo.surf"
+  listen on * port 80
+  block return 302 "https://foo.zone$REQUEST_URI"
+}
+
+server "foo.surf" {
+  alias "www.foo.surf"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/foo.surf.fullchain.pem"
+    key "/etc/ssl/private/foo.surf.key"
+  }
+  block return 302 "https://foo.zone$REQUEST_URI"
+}
+
+server "sidewalk.ninja" {
+  alias "www.sidewalk.ninja"
+  listen on * port 80
+  block return 302 "https://irregular.ninja$REQUEST_URI"
+}
+
+server "sidewalk.ninja" {
+  alias "www.sidewalk.ninja"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/sidewalk.ninja.fullchain.pem"
+    key "/etc/ssl/private/sidewalk.ninja.key"
+  }
+  block return 302 "https://irregular.ninja$REQUEST_URI"
+}
+
+server "buetow.org" {
+  alias "www.buetow.org"
+  listen on * port 80
+  block return 302 "https://foo.zone$REQUEST_URI"
+}
+
+server "paul.buetow.org" {
+  alias "contact.buetow.org"
+  listen on * port 80
+  block return 302 "https://foo.zone/contact-information.html"
+}
+
+server "tmp.buetow.org" {
+  listen on * port 80
+  block return 302 "https://buetow.org/tmp/"
+}
+
+server "buetow.org" {
+  alias "www.buetow.org"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/buetow.org.fullchain.pem"
+    key "/etc/ssl/private/buetow.org.key"
+  }
+  root "/htdocs/buetow.org"
+  location match "/tmp/.*" {
+    directory auto index
+  }
+  location match "/.*" {
+          block return 302 "https://foo.zone$REQUEST_URI"
+  }
+}
+
+server "dtail.dev" {
+  alias "www.dtail.dev"
+  listen on * port 80
+  block return 302 "https://dail.dev"
+}
+
+server "dtail.dev" {
+  alias "www.dtail.dev"
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/dtail.dev.fullchain.pem"
+    key "/etc/ssl/private/dtail.dev.key"
+  }
+  location * {
+    block return 302 "https://github.dtail.dev"
+  }
+}
+
+server "default" {
+  listen on * port 80
+  block return 302 "https://foo.zone$REQUEST_URI"
+}
+
+server "default" {
+  listen on * tls port 443
+  tls {
+    certificate "/etc/ssl/foo.zone.fullchain.pem"
+    key "/etc/ssl/private/foo.zone.key"
+  }
+  block return 302 "https://foo.zone$REQUEST_URI"
+}
diff --git a/frontends/etc/inetd.conf b/frontends/etc/inetd.conf
new file mode 100644
index 0000000..7176af4
--- /dev/null
+++ b/frontends/etc/inetd.conf
@@ -0,0 +1,2 @@
+127.0.0.1:11965 stream tcp nowait www /usr/local/bin/vger vger -v
+*:4242 stream tcp nowait www /bin/cat cat /var/run/ha.status
diff --git a/frontends/etc/mail/aliases b/frontends/etc/mail/aliases
new file mode 100644
index 0000000..91bf1d0
--- /dev/null
+++ b/frontends/etc/mail/aliases
@@ -0,0 +1,103 @@
+#
+#	$OpenBSD: aliases,v 1.68 2020/01/24 06:17:37 tedu Exp $
+#
+#  Aliases in this file will NOT be expanded in the header from
+#  Mail, but WILL be visible over networks or from /usr/libexec/mail.local.
+#
+#	>>>>>>>>>>	The program "newaliases" must be run after
+#	>> NOTE >>	this file is updated for any changes to
+#	>>>>>>>>>>	show through to smtpd.
+#
+
+# Basic system aliases -- these MUST be present
+MAILER-DAEMON: postmaster
+postmaster: root
+
+# General redirections for important pseudo accounts
+daemon:	root
+ftp-bugs: root
+operator: root
+www:	root
+admin:	root
+
+# Redirections for pseudo accounts that should not receive mail
+_bgpd: /dev/null
+_dhcp: /dev/null
+_dpb: /dev/null
+_dvmrpd: /dev/null
+_eigrpd: /dev/null
+_file: /dev/null
+_fingerd: /dev/null
+_ftp: /dev/null
+_hostapd: /dev/null
+_identd: /dev/null
+_iked: /dev/null
+_isakmpd: /dev/null
+_iscsid: /dev/null
+_ldapd: /dev/null
+_ldpd: /dev/null
+_mopd: /dev/null
+_nsd: /dev/null
+_ntp: /dev/null
+_ospfd: /dev/null
+_ospf6d: /dev/null
+_pbuild: /dev/null
+_pfetch: /dev/null
+_pflogd: /dev/null
+_ping: /dev/null
+_pkgfetch: /dev/null
+_pkguntar: /dev/null
+_portmap: /dev/null
+_ppp: /dev/null
+_rad: /dev/null
+_radiusd: /dev/null
+_rbootd: /dev/null
+_relayd: /dev/null
+_ripd: /dev/null
+_rstatd: /dev/null
+_rusersd: /dev/null
+_rwalld: /dev/null
+_smtpd: /dev/null
+_smtpq: /dev/null
+_sndio: /dev/null
+_snmpd: /dev/null
+_spamd: /dev/null
+_switchd: /dev/null
+_syslogd: /dev/null
+_tcpdump: /dev/null
+_traceroute: /dev/null
+_tftpd: /dev/null
+_unbound: /dev/null
+_unwind: /dev/null
+_vmd: /dev/null
+_x11:   /dev/null
+_ypldap: /dev/null
+bin:	/dev/null
+build:	/dev/null
+nobody:	/dev/null
+_tftp_proxy: /dev/null
+_ftp_proxy: /dev/null
+_sndiop: /dev/null
+_syspatch: /dev/null
+_slaacd: /dev/null
+sshd:   /dev/null
+
+# Well-known aliases -- these should be filled in!
+root: paul
+manager: root
+dumper: root
+
+# RFC 2142: NETWORK OPERATIONS MAILBOX NAMES
+abuse:		root
+noc:		root
+security:	root
+
+# RFC 2142: SUPPORT MAILBOX NAMES FOR SPECIFIC INTERNET SERVICES
+hostmaster:	root
+# usenet:	root
+# news:		usenet
+webmaster:	root
+# ftp:		root
+
+paul: paul.buetow@protonmail.com
+albena: albena.buetow@protonmail.com
diff --git a/frontends/etc/mail/smtpd.conf.tpl b/frontends/etc/mail/smtpd.conf.tpl
new file mode 100644
index 0000000..2fb68e3
--- /dev/null
+++ b/frontends/etc/mail/smtpd.conf.tpl
@@ -0,0 +1,23 @@
+# This is the smtpd server system-wide configuration file.
+# See smtpd.conf(5) for more information.
+
+# I used https://www.checktls.com/TestReceiver for testing.
+
+pki "buetow_org_tls" cert "/etc/ssl/buetow.org.fullchain.pem"
+pki "buetow_org_tls" key "/etc/ssl/private/buetow.org.key"
+
+table aliases file:/etc/mail/aliases
+table virtualdomains file:/etc/mail/virtualdomains
+table virtualusers file:/etc/mail/virtualusers
+
+listen on socket
+listen on all tls pki "buetow_org_tls" hostname "<%= $mail_hostname->($hostname) %>"
+#listen on all
+
+action localmail mbox alias <aliases>
+action receive mbox virtual <virtualusers>
+action outbound relay
+
+match from any for domain <virtualdomains> action receive
+match from local for local action localmail
+match from local for any action outbound
diff --git a/frontends/etc/mail/virtualdomains b/frontends/etc/mail/virtualdomains
new file mode 100644
index 0000000..9bdd68a
--- /dev/null
+++ b/frontends/etc/mail/virtualdomains
@@ -0,0 +1,11 @@
+buetow.org
+paul.buetow.org
+mx.buetow.org
+de.buetow.org
+bg.buetow.org
+uk.buetow.org
+us.buetow.org
+es.buetow.org
+dev.buetow.org
+snonux.de
+dtail.dev
diff --git a/frontends/etc/mail/virtualusers b/frontends/etc/mail/virtualusers
new file mode 100644
index 0000000..5ae0b24
--- /dev/null
+++ b/frontends/etc/mail/virtualusers
@@ -0,0 +1,2 @@
+albena@buetow.org albena.buetow@protonmail.com
+@ paul.buetow@protonmail.com
diff --git a/frontends/etc/myname.tpl b/frontends/etc/myname.tpl
new file mode 100644
index 0000000..dcd4ca0
--- /dev/null
+++ b/frontends/etc/myname.tpl
@@ -0,0 +1 @@
+<%= $fqdns->($vio0_ip) %>
diff --git a/frontends/etc/rc.conf.local b/frontends/etc/rc.conf.local
new file mode 100644
index 0000000..842f16d
--- /dev/null
+++ b/frontends/etc/rc.conf.local
@@ -0,0 +1,5 @@
+httpd_flags=
+inetd_flags=
+nsd_flags=
+pkg_scripts="uptimed httpd"
+relayd_flags=
diff --git a/frontends/etc/relayd.conf.tpl b/frontends/etc/relayd.conf.tpl
new file mode 100644
index 0000000..d8553b2
--- /dev/null
+++ b/frontends/etc/relayd.conf.tpl
@@ -0,0 +1,20 @@
+log connection
+
+tcp protocol "gemini" {
+    tls keypair buetow.org
+    tls keypair snonux.de
+    tls keypair foo.zone
+    tls keypair irregular.ninja
+}
+
+relay "gemini4" {
+    listen on <%= $vio0_ip %> port 1965 tls
+    protocol "gemini"
+    forward to 127.0.0.1 port 11965
+}
+
+relay "gemini6" {
+    listen on <%= $ipv6address->($hostname) %> port 1965 tls
+    protocol "gemini"
+    forward to 127.0.0.1 port 11965
+}
diff --git a/frontends/etc/tmux.conf b/frontends/etc/tmux.conf
new file mode 100644
index 0000000..1449326
--- /dev/null
+++ b/frontends/etc/tmux.conf
@@ -0,0 +1,24 @@
+set-option -g allow-rename off
+set-option -g default-terminal "screen-256color"
+set-option -g history-limit 100000
+set-option -g status-bg '#444444'
+set-option -g status-fg '#ffa500'
+
+set-window-option -g mode-keys vi
+
+bind-key h select-pane -L
+bind-key j select-pane -D
+bind-key k select-pane -U
+bind-key l select-pane -R
+
+bind-key H resize-pane -L 5
+bind-key J resize-pane -D 5
+bind-key K resize-pane -U 5
+bind-key L resize-pane -R 5
+
+bind-key b break-pane -d
+bind-key c new-window -c '#{pane_current_path}'
+bind-key p setw synchronize-panes off
+bind-key P setw synchronize-panes on
+bind-key r source-file ~/.tmux.conf \; display-message "~/.tmux.conf reloaded"
+bind-key T choose-tree
diff --git a/frontends/scripts/sitestats.sh b/frontends/scripts/sitestats.sh
new file mode 100755
index 0000000..62702c7
--- /dev/null
+++ b/frontends/scripts/sitestats.sh
@@ -0,0 +1,111 @@
+#!/bin/sh
+
+# This is a quick and dirty script to get some stats for my site.
+# Yes, this could be programmed cleaner, but I wanted to do something quick
+# and dirty and this also with only tools available on the OpenBSD base install.
+
+STATSFILE=/tmp/sitestats.csv
+BOTSFILE=/tmp/sitebots.txt
+TOP=20
+
+header () {
+	echo "proto,host,ip,day,month,time,path"
+}
+
+http_stats () {
+	zgrep -h . /var/www/logs/access.log* |
+	perl -l -n -e 's/\.html/.suffix/; @s=split / +/; next if @s!=11;
+	$s[4]=~s|\[(\d\d)/(...)/\d{4}:(.*)|$1,$2,$3|; print "http,".join ",",@s[0,1,4,7];' 
+}
+
+gemini_stats () {
+	zgrep -h . /var/log/daemon* |
+	perl -l -n -e '@s=split / +/; @v=@s and next if $s[4] eq "vger:";
+	next if !/relayd.*gemini/; ($path) = $v[-1] =~ m|gemini://.*?(/.*)|;
+	next if $path eq ""; $path =~ s/\.gmi/.suffix/;
+	print "gemini,".(split("/", $v[6]))[2].",$s[12],$s[1],$s[0],$s[2],$path"'
+}
+
+parse_logs () {
+	header > $STATSFILE.tmp
+	http_stats >> $STATSFILE.tmp
+	gemini_stats >> $STATSFILE.tmp
+	mv $STATSFILE.tmp $STATSFILE
+}
+
+filter () {
+	# Collect some 'you are a bot' scores.
+	# 1. You visit 2 sites within one single second
+	# 2. You try to call an odd file or path
+	cut -d, -f2,3,6,7 $STATSFILE |
+	perl -l -n -e '($k)=m/(.*?,.*?,.*?),/; $s{$k}++ if /\.suffix/;
+	$s{$k}+=1000 if /(?:target\.suffix|\.php|wordpress|\/wp|\.asp|\.\.|robots\.txt|\.env|\?|\+|%|\*|HNAP1|\/admin\/|\.git\/|microsoft\.exchange|\.lua|\/owa\/)/;
+	END { while (($k,$v) = each %s) { print $k =~ /.*?,(.*?),/ if $v > 1 } }' |
+	sort -u > $BOTSFILE
+
+	# Filte out all bot IPs, also only filter out all known file "types".
+	grep -F -v -f $BOTSFILE $STATSFILE > $STATSFILE.clean1
+	grep -v -E '(proto,host|\.suffix|atom\.xml|\.gif|\.png|\.jpg|,,)' $STATSFILE.clean1 > $STATSFILE.dirt
+	#grep -E '(proto,host|\.suffix|atom\.xml|\.gif|\.png|\.jpg)' $STATSFILE.clean1 > $STATSFILE.clean2
+	mv $STATSFILE.clean1 $STATSFILE
+}
+
+stats () {
+	sed 1d $STATSFILE
+}
+
+top_n () {
+	fields="$1"
+	descr="$2"
+
+	echo "Top $TOP `head -n 1 $STATSFILE | cut -d, -f"$fields"`$descr:"
+ 	cut -d, -f"$fields" | sort | uniq -c | sort -nr | head -n $TOP | sed 's/^/    /'
+	echo
+}
+
+ip_stats () {
+	for proto in http gemini; do 
+		echo -n "Unique $proto IPv4 IPs:\t"
+ 		stats | grep "^$proto," | cut -d, -f3 | grep -F -v : | sort -u | wc -l
+		echo -n "Unique $proto IPv6 IPs:\t" 
+		stats | grep "^$proto," | cut -d, -f3 | grep -F : | sort -u | wc -l
+	done
+}
+
+ip_daily_stats () {
+	echo "Unique IPs by day"
+	for back in $(jot 14); do
+		now=$(date +%s)
+		date=$(date -r $(echo "$now - 86400 * $back" | bc) +%d,%b)
+		echo -n "\t $date:"
+		stats | grep $date | cut -d, -f3 | sort -u | wc -l		
+	done
+}
+
+ip_daily_subscribers () {
+	echo "Unique atom.xml subscribers by day"
+	for back in $(jot 14); do
+		now=$(date +%s)
+		date=$(date -r $(echo "$now - 86400 * $back" | bc) +%d,%b)
+		echo -n "\t $date:"
+		stats | grep $date | grep atom.xml | cut -d, -f3 | sort -u | wc -l		
+	done
+}
+
+main () {
+	date
+	echo
+	parse_logs
+	filter 
+	stats | grep -F .suffix | top_n '1,2,4,5,7' ' (Only content)'
+	stats | top_n 2
+	stats | top_n '4,5'
+	stats | top_n 7
+	stats | grep -F .suffix | top_n 7 ' (Only content)'
+	stats | top_n '1,2,7'
+	ip_stats
+	ip_daily_stats
+	ip_daily_subscribers
+}
+
+main | sed 's/\.suffix//'
diff --git a/frontends/usr/local/bin/ha.pl b/frontends/usr/local/bin/ha.pl
new file mode 100644
index 0000000..5aba2bb
--- /dev/null
+++ b/frontends/usr/local/bin/ha.pl
@@ -0,0 +1,133 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use HTTP::Tiny;
+use IO::Socket::INET;
+use Sys::Hostname;
+use JSON::PP;
+use File::Copy;
+use Data::Dumper;
+
+use constant {
+    STATUS_FILE => '/var/run/ha.status',
+    TMP_STATUS_FILE => '/tmp/ha.status',
+    PARTICIPANTS => qw(blowfish.buetow.org twofish.buetow.org),
+    HA_STATUS_PORT => 4242,
+    MAX_STATUS_AGE => 60,
+}
+
+sub update_ha_status {
+    my @status = @_;
+    my $json = JSON::PP->new->ascii;
+
+    open my $fd, '>', TMP_STATUS_FILE or die $!;
+    print $fd $json->encode($_), "\n" for @status;
+    close $fd;
+
+    copy TMP_STATUS_FILE, STATUS_FILE or die $!;
+    unlink TMP_STATUS_FILE;
+}
+
+sub fetch_remote_ha_status {
+    my $peer = shift;
+    my $socket = new IO::Socket::INET (
+        PeerHost => $peer,
+        PeerPort => HA_STATUS_PORT,
+        Proto => 'tcp',
+    );
+    return undef unless $socket;
+
+    my $response = '';
+    $socket->recv($response, 4096);
+    $socket->close();
+    return split /\n/, $response;
+}
+
+sub check_http_status {
+    my $peer = shift;
+    my $response = HTTP::Tiny->new( max_redirect => 0)->get('http://' . $peer);
+    my $valid_response = $response->{'status'} >= 200 &&
+                         $response->{'status'} < 400;
+    return {
+        endpoint => 'http://' . $peer,
+        peer => $peer,
+        checked_from => hostname,
+        status => $valid_response ? 'OK' : 'ERROR',
+        message => $valid_response ? 'All fine' : 'Got unexpeced response',
+        epoch => time,
+    }
+}
+
+sub check_gemini_status {
+    my $peer = shift;
+    my $socket = new IO::Socket::INET (
+        PeerHost => $peer,
+        PeerPort => 1965,
+        Proto => 'tcp',
+    );
+
+    my $status = {
+        endpoint => 'gemini://' . $peer,
+        peer => $peer,
+        checked_from => hostname,
+        status => $socket ? 'OK' : 'ERROR',
+        message => $socket ? 'All fine' : $!,
+        epoch => time,
+    };
+
+    $socket->close() if $socket;
+    return $status;
+}
+
+sub check_status {
+    my $peer = shift;
+    my @service_status;
+
+    push @service_status, check_http_status $peer;
+    push @service_status, check_gemini_status $peer;
+
+    update_ha_status @service_status;
+    return @service_status;
+}
+
+sub scores {
+    my %scores;
+
+    for my $status (@_) {
+        next if time - $status->{epoch} > MAX_STATUS_AGE;
+        if ($status->{status} eq 'OK') {
+            $scores{$status->{peer}}++;
+        } else {
+            $scores{$status->{peer}} |= 0;
+        }
+    }
+
+    return
+        map { [$_, $scores{$_}] }
+        sort { $scores{$b} <=> $scores{$a} }
+        keys %scores;
+}
+
+sub main {
+    my $json = JSON::PP->new->ascii;
+    my $hostname = hostname;
+    my @all;
+
+    for my $partner (grep { $_ ne $hostname } PARTICIPANTS) {
+        for (check_status $partner) {
+            print $json->encode($_), "\n";
+            push @all, $_;
+        }
+        for (fetch_remote_ha_status $partner) {
+            next if not defined or /^\s*#/;
+            print "$_\n";
+            push @all, $json->decode($_);
+        }
+    }
+
+    print Dumper scores @all;
+}
+
+main;
diff --git a/frontends/var/nsd/etc/key.conf.tpl b/frontends/var/nsd/etc/key.conf.tpl
new file mode 100644
index 0000000..976661a
--- /dev/null
+++ b/frontends/var/nsd/etc/key.conf.tpl
@@ -0,0 +1,4 @@
+key:
+        name: blowfish.buetow.org
+        algorithm: hmac-sha256
+        secret: "<%= $nsd_secret %>"
diff --git a/frontends/var/nsd/etc/nsd.conf.master.tpl b/frontends/var/nsd/etc/nsd.conf.master.tpl
new file mode 100644
index 0000000..310550a
--- /dev/null
+++ b/frontends/var/nsd/etc/nsd.conf.master.tpl
@@ -0,0 +1,19 @@
+include: "/var/nsd/etc/key.conf"
+
+server:
+	hide-version: yes
+	verbosity: 1
+	database: "" # disable database
+	debug-mode: no
+
+remote-control:
+	control-enable: yes
+	control-interface: /var/run/nsd.sock
+
+<% for my $zone (@$dns_zones) { %>
+zone:
+	name: "<%= $zone %>"
+	zonefile: "master/<%= $zone %>.zone"
+	notify: 108.160.134.135 blowfish.buetow.org
+	provide-xfr: 108.160.134.135 blowfish.buetow.org
+<% } %>
diff --git a/frontends/var/nsd/etc/nsd.conf.slave.tpl b/frontends/var/nsd/etc/nsd.conf.slave.tpl
new file mode 100644
index 0000000..d9d93fe
--- /dev/null
+++ b/frontends/var/nsd/etc/nsd.conf.slave.tpl
@@ -0,0 +1,17 @@
+include: "/var/nsd/etc/key.conf"
+
+server:
+	hide-version: yes
+	verbosity: 1
+	database: "" # disable database
+
+remote-control:
+	control-enable: yes
+	control-interface: /var/run/nsd.sock
+
+<% for my $zone (@$dns_zones) { %>
+zone:
+	name: "<%= $zone %>"
+	allow-notify: 23.88.35.144 blowfish.buetow.org
+	request-xfr: 23.88.35.144 blowfish.buetow.org
+<% } %>
diff --git a/frontends/var/nsd/zones/master/buetow.org.zone.tpl b/frontends/var/nsd/zones/master/buetow.org.zone.tpl
new file mode 100644
index 0000000..42bff2d
--- /dev/null
+++ b/frontends/var/nsd/zones/master/buetow.org.zone.tpl
@@ -0,0 +1,38 @@
+$ORIGIN buetow.org.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS    blowfish.buetow.org.
+         IN NS    twofish.buetow.org.
+
+         IN MX 10 buetow.org.
+         IN MX 20 www.buetow.org.
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::42
+
+*        IN MX 10 buetow.org.
+*        IN MX 20 www.buetow.org.
+*        86400 IN A 23.88.35.144
+*        86400 IN AAAA 2a01:4f8:c17:20f1::42
+
+blowfish 86400 IN A 23.88.35.144
+blowfish 86400 IN AAAA 2a01:4f8:c17:20f1::42
+git1     3600 IN CNAME blowfish
+
+twofish  86400 IN A 108.160.134.135
+twofish  86400 IN AAAA 2401:c080:1000:45af:5400:3ff:fec6:ca1d
+git2     3600 IN CNAME twofish
+www      3600 IN CNAME twofish
+
+vulcan   86400 IN A 95.216.174.192
+vulcan   86400 IN AAAA 2a01:4f9:c010:250e::1
+vu       86400 IN CNAME vulcan
+wolke7   3600 IN CNAME vulcan
+edge     3600 IN CNAME vulcan
+
+sofia    86400 IN CNAME 79-100-3-54.ip.btc-net.bg.
+www2     3600 IN CNAME snonux.codeberg.page.
diff --git a/frontends/var/nsd/zones/master/dtail.dev.zone.tpl b/frontends/var/nsd/zones/master/dtail.dev.zone.tpl
new file mode 100644
index 0000000..0d67272
--- /dev/null
+++ b/frontends/var/nsd/zones/master/dtail.dev.zone.tpl
@@ -0,0 +1,16 @@
+$ORIGIN dtail.dev.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::4
+*        86400 IN CNAME blowfish.buetow.org.
+github   86400 IN CNAME mimecast.github.io.
+
diff --git a/frontends/var/nsd/zones/master/foo.surf.zone.tpl b/frontends/var/nsd/zones/master/foo.surf.zone.tpl
new file mode 100644
index 0000000..e92b881
--- /dev/null
+++ b/frontends/var/nsd/zones/master/foo.surf.zone.tpl
@@ -0,0 +1,17 @@
+$ORIGIN foo.surf.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         IN MX 20 buetow.org.
+         IN MX 10 www.buetow.org.
+
+	 86400 IN A 108.160.134.135
+         86400 IN AAAA 2401:c080:1000:45af:5400:3ff:fec6:ca1d
+www      86400 IN CNAME blowfish.buetow.org.
diff --git a/frontends/var/nsd/zones/master/foo.zone.zone.tpl b/frontends/var/nsd/zones/master/foo.zone.zone.tpl
new file mode 100644
index 0000000..4efbf3d
--- /dev/null
+++ b/frontends/var/nsd/zones/master/foo.zone.zone.tpl
@@ -0,0 +1,19 @@
+$ORIGIN foo.zone.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         IN MX 10 buetow.org.
+         IN MX 20 www.buetow.org.
+
+
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::42
+*        86400 IN CNAME blowfish.buetow.org.
+www      86400 IN CNAME twofish.buetow.org.
diff --git a/frontends/var/nsd/zones/master/irregular.ninja.zone.tpl b/frontends/var/nsd/zones/master/irregular.ninja.zone.tpl
new file mode 100644
index 0000000..d3c55e5
--- /dev/null
+++ b/frontends/var/nsd/zones/master/irregular.ninja.zone.tpl
@@ -0,0 +1,18 @@
+$ORIGIN irregular.ninja.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         IN MX 10 buetow.org.
+         IN MX 20 www.buetow.org.
+
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::42
+*        86400 IN CNAME blowfish.buetow.org.
+www      86400 IN CNAME twofish.buetow.org.
diff --git a/frontends/var/nsd/zones/master/sidewalk.ninja.zone.tpl b/frontends/var/nsd/zones/master/sidewalk.ninja.zone.tpl
new file mode 100644
index 0000000..42b1db7
--- /dev/null
+++ b/frontends/var/nsd/zones/master/sidewalk.ninja.zone.tpl
@@ -0,0 +1,18 @@
+$ORIGIN sidewalk.ninja.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         IN MX 10 buetow.org.
+         IN MX 20 www.buetow.org.
+
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::42
+*        86400 IN CNAME blowfish.buetow.org.
+www      86400 IN CNAME twofish.buetow.org.
diff --git a/frontends/var/nsd/zones/master/snonux.de.zone.tpl b/frontends/var/nsd/zones/master/snonux.de.zone.tpl
new file mode 100644
index 0000000..cc530b6
--- /dev/null
+++ b/frontends/var/nsd/zones/master/snonux.de.zone.tpl
@@ -0,0 +1,19 @@
+$ORIGIN snonux.de.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         IN MX 10 buetow.org.
+         IN MX 20 www.buetow.org.
+
+
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::42
+*        86400 IN CNAME blowfish.buetow.org.
+www      86400 IN CNAME twofish.buetow.org.
diff --git a/frontends/var/nsd/zones/master/snonux.me.zone.tpl b/frontends/var/nsd/zones/master/snonux.me.zone.tpl
new file mode 100644
index 0000000..e756998
--- /dev/null
+++ b/frontends/var/nsd/zones/master/snonux.me.zone.tpl
@@ -0,0 +1,19 @@
+$ORIGIN snonux.me.
+$TTL 4h
+@        IN  SOA  blowfish.buetow.org. hostmaster.buetow.org. (
+                  <%= time() %>   ; serial
+                  1h              ; refresh
+                  30m             ; retry
+                  7d              ; expire
+                  1h )            ; negative
+         IN NS   blowfish.buetow.org.
+         IN NS   twofish.buetow.org.
+
+         IN MX 10 buetow.org.
+         IN MX 20 www.buetow.org.
+
+
+         86400 IN A 23.88.35.144
+         86400 IN AAAA 2a01:4f8:c17:20f1::42
+*        86400 IN CNAME blowfish.buetow.org.
+www      86400 IN CNAME twofish.buetow.org.