diff options
| -rw-r--r-- | f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml | 5 | ||||
| -rw-r--r-- | frontends/etc/dserver/dtail-netbsd.json.tpl | 127 | ||||
| -rw-r--r-- | frontends/etc/rc.d/dserver-netbsd.tpl | 34 | ||||
| -rw-r--r-- | frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl | 34 | ||||
| -rw-r--r-- | packages/Makefile | 47 | ||||
| -rw-r--r-- | packages/scripts/pkg-dtail-netbsd.sh | 95 |
6 files changed, 340 insertions, 2 deletions
diff --git a/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml b/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml index 159262c..57ac225 100644 --- a/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml +++ b/f3s/pkgrepo/helm-chart/templates/configmap-nginx.yaml @@ -29,6 +29,11 @@ data: autoindex on; } + # NetBSD pkg_add/pkgin repository + location /netbsd/ { + autoindex on; + } + # Rocky Linux dnf repository location /rockylinux/ { autoindex on; diff --git a/frontends/etc/dserver/dtail-netbsd.json.tpl b/frontends/etc/dserver/dtail-netbsd.json.tpl new file mode 100644 index 0000000..cd99560 --- /dev/null +++ b/frontends/etc/dserver/dtail-netbsd.json.tpl @@ -0,0 +1,127 @@ +{ + "Client": { + "TermColorsEnable": true, + "TermColors": { + "Remote": { + "DelimiterAttr": "Dim", + "DelimiterBg": "Blue", + "DelimiterFg": "Cyan", + "RemoteAttr": "Dim", + "RemoteBg": "Blue", + "RemoteFg": "White", + "CountAttr": "Dim", + "CountBg": "Blue", + "CountFg": "White", + "HostnameAttr": "Bold", + "HostnameBg": "Blue", + "HostnameFg": "White", + "IDAttr": "Dim", + "IDBg": "Blue", + "IDFg": "White", + "StatsOkAttr": "None", + "StatsOkBg": "Green", + "StatsOkFg": "Black", + "StatsWarnAttr": "None", + "StatsWarnBg": "Red", + "StatsWarnFg": "White", + "TextAttr": "None", + "TextBg": "Black", + "TextFg": "White" + }, + "Client": { + "DelimiterAttr": "Dim", + "DelimiterBg": "Yellow", + "DelimiterFg": "Black", + "ClientAttr": "Dim", + "ClientBg": "Yellow", + "ClientFg": "Black", + "HostnameAttr": "Dim", + "HostnameBg": "Yellow", + "HostnameFg": "Black", + "TextAttr": "None", + "TextBg": "Black", + "TextFg": "White" + }, + "Server": { + "DelimiterAttr": "AttrDim", + "DelimiterBg": "BgCyan", + "DelimiterFg": "FgBlack", + "ServerAttr": "AttrDim", + "ServerBg": "BgCyan", + "ServerFg": "FgBlack", + "HostnameAttr": "AttrBold", + "HostnameBg": "BgCyan", + "HostnameFg": "FgBlack", + "TextAttr": "AttrNone", + "TextBg": "BgBlack", + "TextFg": "FgWhite" + }, + "Common": { + "SeverityErrorAttr": "AttrBold", + "SeverityErrorBg": "BgRed", + "SeverityErrorFg": "FgWhite", + "SeverityFatalAttr": "AttrBold", + "SeverityFatalBg": "BgMagenta", + "SeverityFatalFg": "FgWhite", + "SeverityWarnAttr": "AttrBold", + "SeverityWarnBg": "BgBlack", + "SeverityWarnFg": "FgWhite" + }, + "MaprTable": { + "DataAttr": "AttrNone", + "DataBg": "BgBlue", + "DataFg": "FgWhite", + "DelimiterAttr": "AttrDim", + "DelimiterBg": "BgBlue", + "DelimiterFg": "FgWhite", + "HeaderAttr": "AttrBold", + "HeaderBg": "BgBlue", + "HeaderFg": "FgWhite", + "HeaderDelimiterAttr": "AttrDim", + "HeaderDelimiterBg": "BgBlue", + "HeaderDelimiterFg": "FgWhite", + "HeaderSortKeyAttr": "AttrUnderline", + "HeaderGroupKeyAttr": "AttrReverse", + "RawQueryAttr": "AttrDim", + "RawQueryBg": "BgBlack", + "RawQueryFg": "FgCyan" + } + } + }, + "Server": { + "SSHBindAddress": "0.0.0.0", + "HostKeyFile": "/var/run/dserver/cache/ssh_host_key", + "HostKeyBits": 2048, + "MapreduceLogFormat": "default", + "MaxConcurrentCats": 2, + "MaxConcurrentTails": 50, + "MaxConnections": 50, + "MaxLineLength": 1048576, + "Permissions": { + "Default": [ + "readfiles:^/.*$" + ], + "Users": { + "paul": [ + "readfiles:^/.*$" + ], + "pbuetow": [ + "readfiles:^/.*$" + ], + "jamesblake": [ + "readfiles:^/tmp/foo.log$", + "readfiles:^/.*$", + "readfiles:!^/tmp/bar.log$" + ] + } + } + }, + "Common": { + "LogDir": "/var/log/dserver", + "Logger": "Fout", + "LogRotation": "Daily", + "CacheDir": "/var/run/dserver/cache", + "SSHPort": 2222, + "LogLevel": "Info" + } +} diff --git a/frontends/etc/rc.d/dserver-netbsd.tpl b/frontends/etc/rc.d/dserver-netbsd.tpl new file mode 100644 index 0000000..7bd118c --- /dev/null +++ b/frontends/etc/rc.d/dserver-netbsd.tpl @@ -0,0 +1,34 @@ +#!/bin/sh +# +# PROVIDE: dserver +# REQUIRE: DAEMON LOGIN +# KEYWORD: shutdown + +$_rc_subr_loaded . /etc/rc.subr + +name="dserver" +rcvar=$name +command="/usr/local/bin/dserver" + +# dserver does not daemonize itself and NetBSD has no daemon(8) like FreeBSD, +# so background it from the shell; rc.subr finds it again via check_process. +# Logger "Fout" writes the real logs to /var/log/dserver, stdout is minimal. +command_args="-cfg /etc/dserver/dtail.json >> /var/log/dserver/dserver.log 2>&1 &" +dserver_user="dserver" + +start_precmd="dserver_precmd" + +dserver_precmd() +{ + # /var/run is volatile on NetBSD — recreate the runtime dirs and + # repopulate the SSH key cache on every service start. + install -d -o dserver -m 0755 /var/log/dserver + install -d -o dserver -m 0755 /var/run/dserver + install -d -o dserver -m 0755 /var/run/dserver/cache + if [ -x /usr/local/bin/dserver-update-key-cache.sh ]; then + /usr/local/bin/dserver-update-key-cache.sh + fi +} + +load_rc_config $name +run_rc_command "$1" diff --git a/frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl b/frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl new file mode 100644 index 0000000..c50cb72 --- /dev/null +++ b/frontends/scripts/dserver-update-key-cache-netbsd.sh.tpl @@ -0,0 +1,34 @@ +#!/bin/sh +# Refresh the dserver SSH key cache from user authorized_keys files. +# NetBSD variant: called from the dserver rc.d start_precmd (because +# /var/run is volatile across reboots) and from a daily root cron job. + +CACHEDIR=/var/run/dserver/cache +DSERVER_USER=dserver +DSERVER_GROUP=dserver + +echo 'Updating SSH key cache' + +ls /home/ | while read remoteuser; do + keysfile="/home/$remoteuser/.ssh/authorized_keys" + + if [ -f "$keysfile" ]; then + cachefile="$CACHEDIR/$remoteuser.authorized_keys" + echo "Caching $keysfile -> $cachefile" + + cp "$keysfile" "$cachefile" + chown "$DSERVER_USER:$DSERVER_GROUP" "$cachefile" + chmod 600 "$cachefile" + fi +done + +# Remove stale cache entries for users whose authorized_keys no longer exist +find "$CACHEDIR" -name '*.authorized_keys' -type f | while read cachefile; do + remoteuser=$(basename "$cachefile" .authorized_keys) + if [ ! -f "/home/$remoteuser/.ssh/authorized_keys" ]; then + echo "Deleting obsolete cache file $cachefile" + rm "$cachefile" + fi +done + +echo 'All set...' diff --git a/packages/Makefile b/packages/Makefile index 7d85333..860d683 100644 --- a/packages/Makefile +++ b/packages/Makefile @@ -8,6 +8,7 @@ # Multi-binary / CGo packages (built natively on OpenBSD build VM): # make dtail-openbsd # DTail for OpenBSD # make dtail-freebsd # DTail for FreeBSD (cross-compiled, nozstd) +# make dtail-netbsd # DTail for NetBSD/aarch64 (cross-compiled, nozstd; packaged on pi0) # make dtail-rocky # DTail RPM repo for Rocky Linux 9 (x86_64 + aarch64) # # Build VM management: @@ -36,9 +37,14 @@ FREEBSD_SCP := scp -P 22 OPENBSD_HOST := rex@fishfinger.buetow.org OPENBSD_SSH := ssh OPENBSD_SCP := scp -ROCKY_ARM_BUILD_HOST := paul@pi0.lan.buetow.org +# pi0/pi1 were re-imaged to NetBSD; aarch64 RPMs are built on pi2 (Rocky Linux) +ROCKY_ARM_BUILD_HOST := paul@pi2.lan.buetow.org ROCKY_ARM_SSH := ssh -p 22 ROCKY_ARM_SCP := scp -P 22 +# NetBSD packages are assembled natively on pi0 (pkg_create needs a NetBSD host) +NETBSD_BUILD_HOST := paul@pi0.lan.buetow.org +NETBSD_SSH := ssh -p 22 +NETBSD_SCP := scp -P 22 # Local OpenBSD build VM (QEMU/KVM) for native compilation BUILDVM_SSH := ssh -o StrictHostKeyChecking=no -p 2222 @@ -51,6 +57,8 @@ PV_BASE := /data/nfs/k3svolumes/pkgrepo FREEBSD_REPO := freebsd/FreeBSD:15:amd64/latest OPENBSD_VERSION := 7.8 OPENBSD_REPO := openbsd/$(OPENBSD_VERSION)/packages/amd64 +NETBSD_VERSION := 10.1 +NETBSD_REPO := netbsd/$(NETBSD_VERSION)/packages/aarch64 ROCKY_VERSION := 9 ROCKY_REPO := rockylinux/$(ROCKY_VERSION) CONTAINER_ENGINE ?= podman @@ -71,10 +79,13 @@ VERSION = $(shell grep 'Version' $(SRC)/internal/version.go | sed 's/.*"\(.*\)"/ DTAIL_SRC := /home/paul/git/dtail DTAIL_VERSION = $(shell grep 'Version string' $(DTAIL_SRC)/internal/version/version.go | sed 's/.*"\(.*\)"/\1/') DTAIL_BINARIES := dserver dcat dgrep dmap dtail dtailhealth +# NetBSD package versions must not contain dashes (dash separates name from +# version in pkg naming), so 4.3.2-ng becomes 4.3.2ng +DTAIL_NETBSD_VERSION = $(shell echo $(DTAIL_VERSION) | tr -d -) CONF_FRONTENDS := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../frontends) DTAIL_RPM_FILES := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))files/dtail-rocky) -.PHONY: pkg pkg-freebsd pkg-openbsd dtail-openbsd dtail-freebsd dtail-rocky check-vars clean +.PHONY: pkg pkg-freebsd pkg-openbsd dtail-openbsd dtail-freebsd dtail-netbsd dtail-rocky check-vars clean .PHONY: buildvm-start buildvm-stop buildvm-ensure # --- Build VM management --- @@ -207,6 +218,37 @@ dtail-freebsd: rm -rf /tmp/dtail-freebsd-binaries @echo "FreeBSD package dtail-$(DTAIL_VERSION) uploaded to repo" +# Cross-compile dtail for NetBSD/aarch64 (CGO_ENABLED=0, nozstd), package natively +# on pi0 with pkg_create, upload package + pkg_summary.gz to the PV via f0. +# .zst log files are not supported in this binary; gzip still works. +dtail-netbsd: + @echo "Building DTail $(DTAIL_VERSION) for NetBSD/arm64..." + rm -rf /tmp/dtail-netbsd-binaries + mkdir -p /tmp/dtail-netbsd-binaries + cd $(DTAIL_SRC) && for bin in $(DTAIL_BINARIES); do \ + echo " Cross-compiling $$bin for NetBSD..."; \ + CGO_ENABLED=0 GOOS=netbsd GOARCH=arm64 go build -tags nozstd \ + -o /tmp/dtail-netbsd-binaries/$$bin ./cmd/$$bin/main.go || exit 1; \ + done + cp $(CONF_FRONTENDS)/etc/dserver/dtail-netbsd.json.tpl /tmp/dtail-netbsd-binaries/dtail.json + cp $(CONF_FRONTENDS)/scripts/dserver-update-key-cache-netbsd.sh.tpl /tmp/dtail-netbsd-binaries/dserver-update-key-cache.sh + cp $(CONF_FRONTENDS)/etc/rc.d/dserver-netbsd.tpl /tmp/dtail-netbsd-binaries/dserver.rc + @echo "Packaging dtail $(DTAIL_NETBSD_VERSION) for NetBSD on pi0..." + $(NETBSD_SSH) $(NETBSD_BUILD_HOST) "rm -rf /tmp/dtail-netbsd-binaries /tmp/dtail-netbsd-pkg /tmp/pkg-dtail-netbsd.sh" + $(NETBSD_SCP) -r /tmp/dtail-netbsd-binaries $(NETBSD_BUILD_HOST):/tmp/dtail-netbsd-binaries + $(NETBSD_SCP) $(SCRIPTS)/pkg-dtail-netbsd.sh $(NETBSD_BUILD_HOST):/tmp/pkg-dtail-netbsd.sh + $(NETBSD_SSH) $(NETBSD_BUILD_HOST) "/bin/sh /tmp/pkg-dtail-netbsd.sh '$(DTAIL_NETBSD_VERSION)'" + @echo "Copying package to PV via f0..." + $(NETBSD_SCP) $(NETBSD_BUILD_HOST):/tmp/dtail-netbsd-pkg/out/dtail-$(DTAIL_NETBSD_VERSION).tgz /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz + $(NETBSD_SCP) $(NETBSD_BUILD_HOST):/tmp/dtail-netbsd-pkg/out/pkg_summary.gz /tmp/dtail-netbsd-pkg_summary.gz + $(FREEBSD_SCP) /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz $(FREEBSD_HOST):/tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz + $(FREEBSD_SCP) /tmp/dtail-netbsd-pkg_summary.gz $(FREEBSD_HOST):/tmp/dtail-netbsd-pkg_summary.gz + $(FREEBSD_SSH) $(FREEBSD_HOST) "doas mkdir -p $(PV_BASE)/$(NETBSD_REPO) && doas cp /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz $(PV_BASE)/$(NETBSD_REPO)/ && doas cp /tmp/dtail-netbsd-pkg_summary.gz $(PV_BASE)/$(NETBSD_REPO)/pkg_summary.gz && rm /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz /tmp/dtail-netbsd-pkg_summary.gz" + @# Clean up remote and local temp files + $(NETBSD_SSH) $(NETBSD_BUILD_HOST) "rm -rf /tmp/dtail-netbsd-pkg /tmp/dtail-netbsd-binaries /tmp/pkg-dtail-netbsd.sh" + rm -rf /tmp/dtail-netbsd-binaries /tmp/dtail-$(DTAIL_NETBSD_VERSION).tgz /tmp/dtail-netbsd-pkg_summary.gz + @echo "NetBSD package dtail-$(DTAIL_NETBSD_VERSION) uploaded to repo" + # Build RPMs for Rocky Linux 9 (x86_64 + aarch64), generate repodata, and upload to the PV. dtail-rocky: @echo "Packaging dtail $(DTAIL_VERSION) for Rocky Linux $(ROCKY_VERSION)..." @@ -244,4 +286,5 @@ clean: rm -f /tmp/$(NAME)-freebsd /tmp/$(NAME)-openbsd /tmp/$(NAME)-*.tgz rm -rf /tmp/dtail-binaries /tmp/dtail-*.tgz rm -rf /tmp/dtail-freebsd-binaries + rm -rf /tmp/dtail-netbsd-binaries /tmp/dtail-netbsd-pkg_summary.gz rm -rf /tmp/dtail-rocky-repo /tmp/dtail-rocky-upload /tmp/dtail-rocky-aarch64-root diff --git a/packages/scripts/pkg-dtail-netbsd.sh b/packages/scripts/pkg-dtail-netbsd.sh new file mode 100644 index 0000000..f3bc9d6 --- /dev/null +++ b/packages/scripts/pkg-dtail-netbsd.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# Build a NetBSD dtail package from pre-compiled binaries. +# Run on a NetBSD host (e.g. pi0). Called by the Makefile via SSH. +# The .tgz and a matching pkg_summary.gz are left in /tmp/dtail-netbsd-pkg/out/ +# for the Makefile to retrieve. +# +# Note: the pkg_summary.gz only describes the packages built here. If the +# NetBSD repo ever holds more than dtail, regenerate the summary across all +# .tgz files in the repo directory instead. +# +# Arguments: +# $1 — version (NetBSD-safe, no dashes — e.g. 4.3.2ng) + +set -e + +# Non-interactive SSH shells on NetBSD lack /usr/sbin in PATH (pkg_* live there) +PATH=/usr/sbin:/usr/bin:/bin:$PATH +export PATH + +VERSION="$1" +NAME="dtail" +COMMENT="Distributed log tail and grep tool" +DESC="DTail is a distributed DevOps tool for tailing, grepping, catting, and +mapping across many remote machines at once via SSH." + +WORKDIR="/tmp/${NAME}-netbsd-pkg" +rm -rf "$WORKDIR" +mkdir -p \ + "$WORKDIR/stage/usr/local/bin" \ + "$WORKDIR/stage/etc/dserver" \ + "$WORKDIR/stage/etc/rc.d" \ + "$WORKDIR/out" + +# Binaries (cross-compiled linux→netbsd/arm64 with nozstd; .zst logs not supported) +for bin in dserver dcat dgrep dmap dtail dtailhealth; do + cp "/tmp/dtail-netbsd-binaries/${bin}" "$WORKDIR/stage/usr/local/bin/${bin}" + chmod 755 "$WORKDIR/stage/usr/local/bin/${bin}" +done + +# Key cache helper (sh-compatible; walks /home/ on NetBSD) +cp "/tmp/dtail-netbsd-binaries/dserver-update-key-cache.sh" \ + "$WORKDIR/stage/usr/local/bin/dserver-update-key-cache.sh" +chmod 555 "$WORKDIR/stage/usr/local/bin/dserver-update-key-cache.sh" + +# Config (absolute CacheDir/HostKeyFile paths — rc.d starts daemons with cwd /) +cp "/tmp/dtail-netbsd-binaries/dtail.json" "$WORKDIR/stage/etc/dserver/dtail.json" +chmod 644 "$WORKDIR/stage/etc/dserver/dtail.json" + +# rc.d script (NetBSD rc.subr style) +cp "/tmp/dtail-netbsd-binaries/dserver.rc" "$WORKDIR/stage/etc/rc.d/dserver" +chmod 755 "$WORKDIR/stage/etc/rc.d/dserver" + +# Packing list — paths relative to the / install prefix. @owner/@group make +# pkg_add install root-owned files even though staging happens as paul. +cat > "$WORKDIR/plist" <<'PLIST' +@owner root +@group wheel +usr/local/bin/dserver +usr/local/bin/dcat +usr/local/bin/dgrep +usr/local/bin/dmap +usr/local/bin/dtail +usr/local/bin/dtailhealth +usr/local/bin/dserver-update-key-cache.sh +etc/dserver/dtail.json +etc/rc.d/dserver +PLIST + +# Comment and description files +printf '%s\n' "$COMMENT" > "$WORKDIR/comment" +printf '%s\n' "$DESC" > "$WORKDIR/desc" + +# Build info — pkg_add checks these against the target host +cat > "$WORKDIR/build-info" <<BUILDINFO +MACHINE_ARCH=$(uname -p) +OPSYS=NetBSD +OS_VERSION=$(uname -r) +PKGTOOLS_VERSION=$(pkg_info -V) +BUILDINFO + +# Build the package: files are staged under $WORKDIR/stage (-p) but install +# relative to / (-I), mirroring the OpenBSD package layout. +pkg_create \ + -B "$WORKDIR/build-info" \ + -c "$WORKDIR/comment" \ + -d "$WORKDIR/desc" \ + -f "$WORKDIR/plist" \ + -I / \ + -p "$WORKDIR/stage" \ + "$WORKDIR/out/${NAME}-${VERSION}.tgz" + +# Repo metadata for pkgin (pkg_add itself doesn't need it) +( cd "$WORKDIR/out" && pkg_info -X ./*.tgz | gzip -9 > pkg_summary.gz ) + +echo "NetBSD package ${NAME}-${VERSION} built in $WORKDIR/out/" |