summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-07-09 09:37:10 +0300
committerPaul Buetow <paul@buetow.org>2026-07-09 09:37:10 +0300
commit1ea362740236e8e12ea9459a5065f8571a81a99e (patch)
treebdb5f1316e83e8b8253d8b094f4bcb734a6553d6
parent1aa5eb37c626f48bd725578637e5db4ae870ebfe (diff)
packages: add NetBSD dtail pipeline (pi0/pi1, aarch64)
- Makefile target dtail-netbsd: cross-compile netbsd/arm64 (CGO_ENABLED=0, nozstd), package natively on pi0 with pkg_create, upload package plus pkg_summary.gz to the PV at netbsd/10.1/packages/aarch64/ via f0 - scripts/pkg-dtail-netbsd.sh: NetBSD pkg_create packaging (prefix /, root-owned files via @owner/@group plist directives) - frontends: NetBSD dserver rc.d script, dtail.json (absolute cache paths), and key-cache helper templates - pkgrepo nginx: serve /netbsd/ with autoindex - Fix stale ROCKY_ARM_BUILD_HOST: pi0 was re-imaged to NetBSD, aarch64 RPMs are built on pi2 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
-rw-r--r--f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml5
-rw-r--r--frontends/etc/dserver/dtail-netbsd.json.tpl127
-rw-r--r--frontends/etc/rc.d/dserver-netbsd.tpl34
-rw-r--r--frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl34
-rw-r--r--packages/Makefile47
-rw-r--r--packages/scripts/pkg-dtail-netbsd.sh95
6 files changed, 340 insertions, 2 deletions
diff --git a/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml b/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml
index 159262c..57ac225 100644
--- a/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml
+++ b/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml
@@ -29,6 +29,11 @@ data:
autoindex on;
}
+ # NetBSD pkg_add/pkgin repository
+ location /netbsd/ {
+ autoindex on;
+ }
+
# Rocky Linux dnf repository
location /rockylinux/ {
autoindex on;
diff --git a/frontends/etc/dserver/dtail-netbsd.json.tpl b/frontends/etc/dserver/dtail-netbsd.json.tpl
new file mode 100644
index 0000000..cd99560
--- /dev/null
+++ b/frontends/etc/dserver/dtail-netbsd.json.tpl
@@ -0,0 +1,127 @@
+{
+ "Client": {
+ "TermColorsEnable": true,
+ "TermColors": {
+ "Remote": {
+ "DelimiterAttr": "Dim",
+ "DelimiterBg": "Blue",
+ "DelimiterFg": "Cyan",
+ "RemoteAttr": "Dim",
+ "RemoteBg": "Blue",
+ "RemoteFg": "White",
+ "CountAttr": "Dim",
+ "CountBg": "Blue",
+ "CountFg": "White",
+ "HostnameAttr": "Bold",
+ "HostnameBg": "Blue",
+ "HostnameFg": "White",
+ "IDAttr": "Dim",
+ "IDBg": "Blue",
+ "IDFg": "White",
+ "StatsOkAttr": "None",
+ "StatsOkBg": "Green",
+ "StatsOkFg": "Black",
+ "StatsWarnAttr": "None",
+ "StatsWarnBg": "Red",
+ "StatsWarnFg": "White",
+ "TextAttr": "None",
+ "TextBg": "Black",
+ "TextFg": "White"
+ },
+ "Client": {
+ "DelimiterAttr": "Dim",
+ "DelimiterBg": "Yellow",
+ "DelimiterFg": "Black",
+ "ClientAttr": "Dim",
+ "ClientBg": "Yellow",
+ "ClientFg": "Black",
+ "HostnameAttr": "Dim",
+ "HostnameBg": "Yellow",
+ "HostnameFg": "Black",
+ "TextAttr": "None",
+ "TextBg": "Black",
+ "TextFg": "White"
+ },
+ "Server": {
+ "DelimiterAttr": "AttrDim",
+ "DelimiterBg": "BgCyan",
+ "DelimiterFg": "FgBlack",
+ "ServerAttr": "AttrDim",
+ "ServerBg": "BgCyan",
+ "ServerFg": "FgBlack",
+ "HostnameAttr": "AttrBold",
+ "HostnameBg": "BgCyan",
+ "HostnameFg": "FgBlack",
+ "TextAttr": "AttrNone",
+ "TextBg": "BgBlack",
+ "TextFg": "FgWhite"
+ },
+ "Common": {
+ "SeverityErrorAttr": "AttrBold",
+ "SeverityErrorBg": "BgRed",
+ "SeverityErrorFg": "FgWhite",
+ "SeverityFatalAttr": "AttrBold",
+ "SeverityFatalBg": "BgMagenta",
+ "SeverityFatalFg": "FgWhite",
+ "SeverityWarnAttr": "AttrBold",
+ "SeverityWarnBg": "BgBlack",
+ "SeverityWarnFg": "FgWhite"
+ },
+ "MaprTable": {
+ "DataAttr": "AttrNone",
+ "DataBg": "BgBlue",
+ "DataFg": "FgWhite",
+ "DelimiterAttr": "AttrDim",
+ "DelimiterBg": "BgBlue",
+ "DelimiterFg": "FgWhite",
+ "HeaderAttr": "AttrBold",
+ "HeaderBg": "BgBlue",
+ "HeaderFg": "FgWhite",
+ "HeaderDelimiterAttr": "AttrDim",
+ "HeaderDelimiterBg": "BgBlue",
+ "HeaderDelimiterFg": "FgWhite",
+ "HeaderSortKeyAttr": "AttrUnderline",
+ "HeaderGroupKeyAttr": "AttrReverse",
+ "RawQueryAttr": "AttrDim",
+ "RawQueryBg": "BgBlack",
+ "RawQueryFg": "FgCyan"
+ }
+ }
+ },
+ "Server": {
+ "SSHBindAddress": "0.0.0.0",
+ "HostKeyFile": "/var/run/dserver/cache/ssh_host_key",
+ "HostKeyBits": 2048,
+ "MapreduceLogFormat": "default",
+ "MaxConcurrentCats": 2,
+ "MaxConcurrentTails": 50,
+ "MaxConnections": 50,
+ "MaxLineLength": 1048576,
+ "Permissions": {
+ "Default": [
+ "readfiles:^/.*$"
+ ],
+ "Users": {
+ "paul": [
+ "readfiles:^/.*$"
+ ],
+ "pbuetow": [
+ "readfiles:^/.*$"
+ ],
+ "jamesblake": [
+ "readfiles:^/tmp/foo.log$",
+ "readfiles:^/.*$",
+ "readfiles:!^/tmp/bar.log$"
+ ]
+ }
+ }
+ },
+ "Common": {
+ "LogDir": "/var/log/dserver",
+ "Logger": "Fout",
+ "LogRotation": "Daily",
+ "CacheDir": "/var/run/dserver/cache",
+ "SSHPort": 2222,
+ "LogLevel": "Info"
+ }
+}
diff --git a/frontends/etc/rc.d/dserver-netbsd.tpl b/frontends/etc/rc.d/dserver-netbsd.tpl
new file mode 100644
index 0000000..7bd118c
--- /dev/null
+++ b/frontends/etc/rc.d/dserver-netbsd.tpl
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# PROVIDE: dserver
+# REQUIRE: DAEMON LOGIN
+# KEYWORD: shutdown
+
+$_rc_subr_loaded . /etc/rc.subr
+
+name="dserver"
+rcvar=$name
+command="/usr/local/bin/dserver"
+
+# dserver does not daemonize itself and NetBSD has no daemon(8) like FreeBSD,
+# so background it from the shell; rc.subr finds it again via check_process.
+# Logger "Fout" writes the real logs to /var/log/dserver, stdout is minimal.
+command_args="-cfg /etc/dserver/dtail.json >> /var/log/dserver/dserver.log 2>&1 &"
+dserver_user="dserver"
+
+start_precmd="dserver_precmd"
+
+dserver_precmd()
+{
+ # /var/run is volatile on NetBSD — recreate the runtime dirs and
+ # repopulate the SSH key cache on every service start.
+ install -d -o dserver -m 0755 /var/log/dserver
+ install -d -o dserver -m 0755 /var/run/dserver
+ install -d -o dserver -m 0755 /var/run/dserver/cache
+ if [ -x /usr/local/bin/dserver-update-key-cache.sh ]; then
+ /usr/local/bin/dserver-update-key-cache.sh
+ fi
+}
+
+load_rc_config $name
+run_rc_command "$1"
diff --git a/frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl b/frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl
new file mode 100644
index 0000000..c50cb72
--- /dev/null
+++ b/frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl
@@ -0,0 +1,34 @@
+#!/bin/sh
+# Refresh the dserver SSH key cache from user authorized_keys files.
+# NetBSD variant: called from the dserver rc.d start_precmd (because
+# /var/run is volatile across reboots) and from a daily root cron job.
+
+CACHEDIR=/var/run/dserver/cache
+DSERVER_USER=dserver
+DSERVER_GROUP=dserver
+
+echo 'Updating SSH key cache'
+
+ls /home/ | while read remoteuser; do
+ keysfile="/home/$remoteuser/.ssh/authorized_keys"
+
+ if [ -f "$keysfile" ]; then
+ cachefile="$CACHEDIR/$remoteuser.authorized_keys"
+ echo "Caching $keysfile -> $cachefile"
+
+ cp "$keysfile" "$cachefile"
+ chown "$DSERVER_USER:$DSERVER_GROUP" "$cachefile"
+ chmod 600 "$cachefile"
+ fi
+done
+
+# Remove stale cache entries for users whose authorized_keys no longer exist
+find "$CACHEDIR" -name '*.authorized_keys' -type f | while read cachefile; do
+ remoteuser=$(basename "$cachefile" .authorized_keys)
+ if [ ! -f "/home/$remoteuser/.ssh/authorized_keys" ]; then
+ echo "Deleting obsolete cache file $cachefile"
+ rm "$cachefile"
+ fi
+done
+
+echo 'All set...'
diff --git a/packages/Makefile b/packages/Makefile
index 7d85333..860d683 100644
--- a/packages/Makefile
+++ b/packages/Makefile
@@ -8,6 +8,7 @@
# Multi-binary / CGo packages (built natively on OpenBSD build VM):
# make dtail-openbsd # DTail for OpenBSD
# make dtail-freebsd # DTail for FreeBSD (cross-compiled, nozstd)
+# make dtail-netbsd # DTail for NetBSD/aarch64 (cross-compiled, nozstd; packaged on pi0)
# make dtail-rocky # DTail RPM repo for Rocky Linux 9 (x86_64 + aarch64)
#
# Build VM management:
@@ -36,9 +37,14 @@ FREEBSD_SCP := scp -P 22
OPENBSD_HOST := rex@fishfinger.buetow.org
OPENBSD_SSH := ssh
OPENBSD_SCP := scp
-ROCKY_ARM_BUILD_HOST := paul@pi0.lan.buetow.org
+# pi0/pi1 were re-imaged to NetBSD; aarch64 RPMs are built on pi2 (Rocky Linux)
+ROCKY_ARM_BUILD_HOST := paul@pi2.lan.buetow.org
ROCKY_ARM_SSH := ssh -p 22
ROCKY_ARM_SCP := scp -P 22
+# NetBSD packages are assembled natively on pi0 (pkg_create needs a NetBSD host)
+NETBSD_BUILD_HOST := paul@pi0.lan.buetow.org
+NETBSD_SSH := ssh -p 22
+NETBSD_SCP := scp -P 22
# Local OpenBSD build VM (QEMU/KVM) for native compilation
BUILDVM_SSH := ssh -o StrictHostKeyChecking=no -p 2222
@@ -51,6 +57,8 @@ PV_BASE := /data/nfs/k3svolumes/pkgrepo
FREEBSD_REPO := freebsd/FreeBSD:15:amd64/latest
OPENBSD_VERSION := 7.8
OPENBSD_REPO := openbsd/$(OPENBSD_VERSION)/packages/amd64
+NETBSD_VERSION := 10.1
+NETBSD_REPO := netbsd/$(NETBSD_VERSION)/packages/aarch64
ROCKY_VERSION := 9
ROCKY_REPO := rockylinux/$(ROCKY_VERSION)
CONTAINER_ENGINE ?= podman
@@ -71,10 +79,13 @@ VERSION = $(shell grep 'Version' $(SRC)/internal/version.go | sed 's/.*"\(.*\)"/
DTAIL_SRC := /home/paul/git/dtail
DTAIL_VERSION = $(shell grep 'Version string' $(DTAIL_SRC)/internal/version/version.go | sed 's/.*"\(.*\)"/\1/')
DTAIL_BINARIES := dserver dcat dgrep dmap dtail dtailhealth
+# NetBSD package versions must not contain dashes (dash separates name from
+# version in pkg naming), so 4.3.2-ng becomes 4.3.2ng
+DTAIL_NETBSD_VERSION = $(shell echo $(DTAIL_VERSION) | tr -d -)
CONF_FRONTENDS := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../frontends)
DTAIL_RPM_FILES := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))files/dtail-rocky)
-.PHONY: pkg pkg-freebsd pkg-openbsd dtail-openbsd dtail-freebsd dtail-rocky check-vars clean
+.PHONY: pkg pkg-freebsd pkg-openbsd dtail-openbsd dtail-freebsd dtail-netbsd dtail-rocky check-vars clean
.PHONY: buildvm-start buildvm-stop buildvm-ensure
# --- Build VM management ---
@@ -207,6 +218,37 @@ dtail-freebsd:
rm -rf /tmp/dtail-freebsd-binaries
@echo "FreeBSD package dtail-$(DTAIL_VERSION) uploaded to repo"
+# Cross-compile dtail for NetBSD/aarch64 (CGO_ENABLED=0, nozstd), package natively
+# on pi0 with pkg_create, upload package + pkg_summary.gz to the PV via f0.
+# .zst log files are not supported in this binary; gzip still works.
+dtail-netbsd:
+ @echo "Building DTail $(DTAIL_VERSION) for NetBSD/arm64..."
+ rm -rf /tmp/dtail-netbsd-binaries
+ mkdir -p /tmp/dtail-netbsd-binaries
+ cd $(DTAIL_SRC) && for bin in $(DTAIL_BINARIES); do \
+ echo " Cross-compiling $$bin for NetBSD..."; \
+ CGO_ENABLED=0 GOOS=netbsd GOARCH=arm64 go build -tags nozstd \
+ -o /tmp/dtail-netbsd-binaries/$$bin ./cmd/$$bin/main.go || exit 1; \
+ done
+ cp $(CONF_FRONTENDS)/etc/dserver/dtail-netbsd.json.tpl /tmp/dtail-netbsd-binaries/dtail.json
+ cp $(CONF_FRONTENDS)/scripts/dserver-update-key-cache-netbsd.sh.tpl /tmp/dtail-netbsd-binaries/dserver-update-key-cache.sh
+ cp $(CONF_FRONTENDS)/etc/rc.d/dserver-netbsd.tpl /tmp/dtail-netbsd-binaries/dserver.rc
+ @echo "Packaging dtail $(DTAIL_NETBSD_VERSION) for NetBSD on pi0..."
+ $(NETBSD_SSH) $(NETBSD_BUILD_HOST) "rm -rf /tmp/dtail-netbsd-binaries /tmp/dtail-netbsd-pkg /tmp/pkg-dtail-netbsd.sh"
+ $(NETBSD_SCP) -r /tmp/dtail-netbsd-binaries $(NETBSD_BUILD_HOST):/tmp/dtail-netbsd-binaries
+ $(NETBSD_SCP) $(SCRIPTS)/pkg-dtail-netbsd.sh $(NETBSD_BUILD_HOST):/tmp/pkg-dtail-netbsd.sh
+ $(NETBSD_SSH) $(NETBSD_BUILD_HOST) "/bin/sh /tmp/pkg-dtail-netbsd.sh '$(DTAIL_NETBSD_VERSION)'"
+ @echo "Copying package to PV via f0..."
+ $(NETBSD_SCP) $(NETBSD_BUILD_HOST):/tmp/dtail-netbsd-pkg/out/dtail-$(DTAIL_NETBSD_VERSION).tgz /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz
+ $(NETBSD_SCP) $(NETBSD_BUILD_HOST):/tmp/dtail-netbsd-pkg/out/pkg_summary.gz /tmp/dtail-netbsd-pkg_summary.gz
+ $(FREEBSD_SCP) /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz $(FREEBSD_HOST):/tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz
+ $(FREEBSD_SCP) /tmp/dtail-netbsd-pkg_summary.gz $(FREEBSD_HOST):/tmp/dtail-netbsd-pkg_summary.gz
+ $(FREEBSD_SSH) $(FREEBSD_HOST) "doas mkdir -p $(PV_BASE)/$(NETBSD_REPO) && doas cp /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz $(PV_BASE)/$(NETBSD_REPO)/ && doas cp /tmp/dtail-netbsd-pkg_summary.gz $(PV_BASE)/$(NETBSD_REPO)/pkg_summary.gz && rm /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz /tmp/dtail-netbsd-pkg_summary.gz"
+ @# Clean up remote and local temp files
+ $(NETBSD_SSH) $(NETBSD_BUILD_HOST) "rm -rf /tmp/dtail-netbsd-pkg /tmp/dtail-netbsd-binaries /tmp/pkg-dtail-netbsd.sh"
+ rm -rf /tmp/dtail-netbsd-binaries /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz /tmp/dtail-netbsd-pkg_summary.gz
+ @echo "NetBSD package dtail-$(DTAIL_NETBSD_VERSION) uploaded to repo"
+
# Build RPMs for Rocky Linux 9 (x86_64 + aarch64), generate repodata, and upload to the PV.
dtail-rocky:
@echo "Packaging dtail $(DTAIL_VERSION) for Rocky Linux $(ROCKY_VERSION)..."
@@ -244,4 +286,5 @@ clean:
rm -f /tmp/$(NAME)-freebsd /tmp/$(NAME)-openbsd /tmp/$(NAME)-*.tgz
rm -rf /tmp/dtail-binaries /tmp/dtail-*.tgz
rm -rf /tmp/dtail-freebsd-binaries
+ rm -rf /tmp/dtail-netbsd-binaries /tmp/dtail-netbsd-pkg_summary.gz
rm -rf /tmp/dtail-rocky-repo /tmp/dtail-rocky-upload /tmp/dtail-rocky-aarch64-root
diff --git a/packages/scripts/pkg-dtail-netbsd.sh b/packages/scripts/pkg-dtail-netbsd.sh
new file mode 100644
index 0000000..f3bc9d6
--- /dev/null
+++ b/packages/scripts/pkg-dtail-netbsd.sh
@@ -0,0 +1,95 @@
+#!/bin/sh
+# Build a NetBSD dtail package from pre-compiled binaries.
+# Run on a NetBSD host (e.g. pi0). Called by the Makefile via SSH.
+# The .tgz and a matching pkg_summary.gz are left in /tmp/dtail-netbsd-pkg/out/
+# for the Makefile to retrieve.
+#
+# Note: the pkg_summary.gz only describes the packages built here. If the
+# NetBSD repo ever holds more than dtail, regenerate the summary across all
+# .tgz files in the repo directory instead.
+#
+# Arguments:
+# $1 — version (NetBSD-safe, no dashes — e.g. 4.3.2ng)
+
+set -e
+
+# Non-interactive SSH shells on NetBSD lack /usr/sbin in PATH (pkg_* live there)
+PATH=/usr/sbin:/usr/bin:/bin:$PATH
+export PATH
+
+VERSION="$1"
+NAME="dtail"
+COMMENT="Distributed log tail and grep tool"
+DESC="DTail is a distributed DevOps tool for tailing, grepping, catting, and
+mapping across many remote machines at once via SSH."
+
+WORKDIR="/tmp/${NAME}-netbsd-pkg"
+rm -rf "$WORKDIR"
+mkdir -p \
+ "$WORKDIR/stage/usr/local/bin" \
+ "$WORKDIR/stage/etc/dserver" \
+ "$WORKDIR/stage/etc/rc.d" \
+ "$WORKDIR/out"
+
+# Binaries (cross-compiled linux→netbsd/arm64 with nozstd; .zst logs not supported)
+for bin in dserver dcat dgrep dmap dtail dtailhealth; do
+ cp "/tmp/dtail-netbsd-binaries/${bin}" "$WORKDIR/stage/usr/local/bin/${bin}"
+ chmod 755 "$WORKDIR/stage/usr/local/bin/${bin}"
+done
+
+# Key cache helper (sh-compatible; walks /home/ on NetBSD)
+cp "/tmp/dtail-netbsd-binaries/dserver-update-key-cache.sh" \
+ "$WORKDIR/stage/usr/local/bin/dserver-update-key-cache.sh"
+chmod 555 "$WORKDIR/stage/usr/local/bin/dserver-update-key-cache.sh"
+
+# Config (absolute CacheDir/HostKeyFile paths — rc.d starts daemons with cwd /)
+cp "/tmp/dtail-netbsd-binaries/dtail.json" "$WORKDIR/stage/etc/dserver/dtail.json"
+chmod 644 "$WORKDIR/stage/etc/dserver/dtail.json"
+
+# rc.d script (NetBSD rc.subr style)
+cp "/tmp/dtail-netbsd-binaries/dserver.rc" "$WORKDIR/stage/etc/rc.d/dserver"
+chmod 755 "$WORKDIR/stage/etc/rc.d/dserver"
+
+# Packing list — paths relative to the / install prefix. @owner/@group make
+# pkg_add install root-owned files even though staging happens as paul.
+cat > "$WORKDIR/plist" <<'PLIST'
+@owner root
+@group wheel
+usr/local/bin/dserver
+usr/local/bin/dcat
+usr/local/bin/dgrep
+usr/local/bin/dmap
+usr/local/bin/dtail
+usr/local/bin/dtailhealth
+usr/local/bin/dserver-update-key-cache.sh
+etc/dserver/dtail.json
+etc/rc.d/dserver
+PLIST
+
+# Comment and description files
+printf '%s\n' "$COMMENT" > "$WORKDIR/comment"
+printf '%s\n' "$DESC" > "$WORKDIR/desc"
+
+# Build info — pkg_add checks these against the target host
+cat > "$WORKDIR/build-info" <<BUILDINFO
+MACHINE_ARCH=$(uname -p)
+OPSYS=NetBSD
+OS_VERSION=$(uname -r)
+PKGTOOLS_VERSION=$(pkg_info -V)
+BUILDINFO
+
+# Build the package: files are staged under $WORKDIR/stage (-p) but install
+# relative to / (-I), mirroring the OpenBSD package layout.
+pkg_create \
+ -B "$WORKDIR/build-info" \
+ -c "$WORKDIR/comment" \
+ -d "$WORKDIR/desc" \
+ -f "$WORKDIR/plist" \
+ -I / \
+ -p "$WORKDIR/stage" \
+ "$WORKDIR/out/${NAME}-${VERSION}.tgz"
+
+# Repo metadata for pkgin (pkg_add itself doesn't need it)
+( cd "$WORKDIR/out" && pkg_info -X ./*.tgz | gzip -9 > pkg_summary.gz )
+
+echo "NetBSD package ${NAME}-${VERSION} built in $WORKDIR/out/"