f3s: add robust USB key mounting

5 files changed, 364 insertions, 0 deletions

diff --git a/f3s/freebsd-hosts/carp/carpcontrol.sh b/f3s/freebsd-hosts/carp/carpcontrol.sh
new file mode 100644
index 0000000..12f0c6b
--- /dev/null
+++ b/f3s/freebsd-hosts/carp/carpcontrol.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+# CARP state change control script
+
+HOSTNAME=`hostname`
+
+if [ ! -f /data/nfs/nfs.DO_NOT_REMOVE ]; then
+    logger '/data/nfs not mounted, mounting it now!'
+    if ! /usr/local/sbin/f3s-mount-keys; then
+        logger 'could not mount /keys; refusing to load ZFS keys for NFS'
+        exit 1
+    fi
+    if [ "$HOSTNAME" = 'f0.lan.buetow.org' ]; then
+        zfs load-key -L file:///keys/f0.lan.buetow.org:zdata.key zdata/enc/nfsdata
+        zfs set mountpoint=/data/nfs zdata/enc/nfsdata
+    else
+        zfs load-key -L file:///keys/f0.lan.buetow.org:zdata.key zdata/sink/f0/zdata/enc/nfsdata
+        zfs set mountpoint=/data/nfs zdata/sink/f0/zdata/enc/nfsdata
+        zfs mount zdata/sink/f0/zdata/enc/nfsdata
+        zfs set readonly=on zdata/sink/f0/zdata/enc/nfsdata
+    fi
+    service nfsd stop 2>&1
+    service mountd stop 2>&1
+fi
+
+case "$2" in
+    MASTER)
+        logger "CARP state changed to MASTER, starting services"
+        service rpcbind start >/dev/null 2>&1
+        service mountd start >/dev/null 2>&1
+        service nfsd start >/dev/null 2>&1
+        service nfsuserd start >/dev/null 2>&1
+        service stunnel restart >/dev/null 2>&1
+        logger "CARP MASTER: NFS and stunnel services started"
+        ;;
+    BACKUP)
+        logger "CARP state changed to BACKUP, stopping services"
+        service stunnel stop >/dev/null 2>&1
+        service nfsd stop >/dev/null 2>&1
+        service mountd stop >/dev/null 2>&1
+        service nfsuserd stop >/dev/null 2>&1
+        logger "CARP BACKUP: NFS and stunnel services stopped"
+        ;;
+    *)
+        logger "CARP state changed to $2 (unhandled)"
+        ;;
+esac
diff --git a/f3s/freebsd-hosts/keys/README.md b/f3s/freebsd-hosts/keys/README.md
new file mode 100644
index 0000000..23c5875
--- /dev/null
+++ b/f3s/freebsd-hosts/keys/README.md
@@ -0,0 +1,91 @@
+# f3s FreeBSD USB ZFS Keys
+
+The f-hosts keep ZFS raw encryption keys on a UFS USB stick mounted at `/keys`.
+Do not mount that stick from `/etc/fstab`: a missing or corrupt USB stick must
+not block the FreeBSD base OS from booting.
+
+## Installed Files
+
+- `/usr/local/sbin/f3s-mount-keys` mounts the USB key stick on demand.
+- `/usr/local/sbin/f3s-load-zfs-keys` is a manual recovery helper that mounts
+  `/keys`, then loads datasets from `zfskeys_datasets`.
+- `/etc/rc.d/f3skeys` runs before FreeBSD's built-in `zfskeys` service.
+
+The rc service deliberately returns success if the USB stick is missing or
+fails `fsck_ufs -p`. Boot continues; encrypted datasets remain locked until the
+stick is repaired and keys are loaded manually.
+
+## Host Configuration
+
+On each f-host:
+
+```sh
+doas install -o root -g wheel -m 0555 f3s-mount-keys /usr/local/sbin/f3s-mount-keys
+doas install -o root -g wheel -m 0555 f3s-load-zfs-keys /usr/local/sbin/f3s-load-zfs-keys
+doas install -o root -g wheel -m 0555 f3skeys.rc /etc/rc.d/f3skeys
+doas sysrc f3skeys_enable=YES
+doas sysrc zfskeys_enable=YES
+```
+
+Comment out any `/keys` line in `/etc/fstab`, for example:
+
+```fstab
+# /dev/da0 /keys ufs rw 0 2
+```
+
+If possible, label the UFS filesystem `F3S_KEYS` and let the script mount
+`/dev/ufs/F3S_KEYS`. The script still falls back to `/dev/da0` for the current
+single-stick host layout.
+
+Current boot key-load datasets:
+
+```sh
+# f0
+doas sysrc zfskeys_datasets="zdata/enc zdata/enc/nfsdata zroot/bhyve zroot/garage"
+
+# f1
+doas sysrc zfskeys_datasets="zdata/enc zroot/bhyve zroot/garage zdata/sink/f0/zdata/enc/nfsdata"
+
+# f2
+doas sysrc zfskeys_datasets="zdata/enc zroot/bhyve zroot/garage zroot/sink/f3/zroot/bhyve/freebsd"
+
+# f3
+doas sysrc zfskeys_datasets="zroot/bhyve"
+```
+
+Replicated encrypted sinks use file keylocations so boot can load them without
+a prompt:
+
+```sh
+# f1
+doas zfs set keylocation=file:///keys/f0.lan.buetow.org:zdata.key \
+  zdata/sink/f0/zdata/enc/nfsdata
+
+# f2
+doas zfs set keylocation=file:///keys/f3.lan.buetow.org:bhyve.key \
+  zroot/sink/f3/zroot/bhyve/freebsd
+```
+
+## Verification
+
+```sh
+doas service f3skeys start
+mount | grep ' /keys '
+doas service zfskeys status
+doas /usr/local/sbin/f3s-load-zfs-keys
+rcorder /etc/rc.d/* /usr/local/etc/rc.d/* | grep -E 'f3skeys|zfskeys|zfs$'
+```
+
+After a reboot, verify:
+
+```sh
+mount | grep ' /keys '
+sysrc -n f3skeys_enable
+sysrc -n zfskeys_enable
+sysrc -n zfskeys_datasets
+zfs list -H -o name,encryption,keylocation,keystatus,mounted |
+  awk '$2 != "off" { print }'
+```
+
+`zroot/sink/f3/zroot/bhyve/freebsd` on f2 has `mountpoint=none`; its key should
+be available after boot, but the dataset is not expected to be mounted.
diff --git a/f3s/freebsd-hosts/keys/f3s-load-zfs-keys b/f3s/freebsd-hosts/keys/f3s-load-zfs-keys
new file mode 100644
index 0000000..e3db82e
--- /dev/null
+++ b/f3s/freebsd-hosts/keys/f3s-load-zfs-keys
@@ -0,0 +1,88 @@
+#!/bin/sh
+
+set -u
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
+LOGTAG="f3s-load-zfs-keys"
+TIMEOUT="${F3S_ZFSKEYS_TIMEOUT:-10}"
+
+log()
+{
+    logger -t "$LOGTAG" "$*"
+    echo "$LOGTAG: $*"
+}
+
+datasets_from_rc()
+{
+    sysrc -n zfskeys_datasets 2>/dev/null || true
+}
+
+load_one()
+{
+    dataset="$1"
+
+    if ! zfs list -H "$dataset" >/dev/null 2>&1; then
+        log "dataset does not exist: $dataset"
+        return 0
+    fi
+
+    keystatus="$(zfs get -H -o value keystatus "$dataset" 2>/dev/null || true)"
+    if [ "$keystatus" = "available" ]; then
+        log "key already loaded for $dataset"
+        return 0
+    fi
+
+    keylocation="$(zfs get -H -o value keylocation "$dataset" 2>/dev/null || true)"
+    case "$keylocation" in
+        file://*)
+            keyfile="${keylocation#file://}"
+            if [ ! -s "$keyfile" ] || [ ! -r "$keyfile" ]; then
+                log "key file is missing or unreadable for $dataset: $keyfile"
+                return 1
+            fi
+            ;;
+        *)
+            log "keylocation for $dataset is not a file key: $keylocation"
+            return 0
+            ;;
+    esac
+
+    if ! zfs load-key -n "$dataset" >/dev/null 2>&1; then
+        log "key verification failed for $dataset"
+        return 1
+    fi
+
+    if timeout "$TIMEOUT" zfs load-key "$dataset"; then
+        log "loaded key for $dataset"
+    else
+        log "failed to load key for $dataset"
+        return 1
+    fi
+
+    mounted="$(zfs get -H -o value mounted "$dataset" 2>/dev/null || true)"
+    canmount="$(zfs get -H -o value canmount "$dataset" 2>/dev/null || true)"
+    if [ "$mounted" = "no" ] && [ "$canmount" != "off" ] && [ "$canmount" != "noauto" ]; then
+        zfs mount "$dataset" >/dev/null 2>&1 || true
+    fi
+}
+
+/usr/local/sbin/f3s-mount-keys --strict || exit 1
+
+if [ "$#" -gt 0 ]; then
+    DATASETS="$*"
+else
+    DATASETS="$(datasets_from_rc)"
+fi
+
+if [ -z "$DATASETS" ]; then
+    log "no datasets supplied and zfskeys_datasets is empty"
+    exit 0
+fi
+
+rc=0
+for dataset in $DATASETS; do
+    load_one "$dataset" || rc=1
+done
+
+exit "$rc"
diff --git a/f3s/freebsd-hosts/keys/f3s-mount-keys b/f3s/freebsd-hosts/keys/f3s-mount-keys
new file mode 100644
index 0000000..c72109d
--- /dev/null
+++ b/f3s/freebsd-hosts/keys/f3s-mount-keys
@@ -0,0 +1,110 @@
+#!/bin/sh
+
+set -u
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
+MOUNTPOINT="${F3S_KEYS_MOUNTPOINT:-/keys}"
+LABEL="${F3S_KEYS_LABEL:-F3S_KEYS}"
+DEVICES="${F3S_KEYS_DEVICES:-/dev/ufs/${LABEL} /dev/da0}"
+WAIT_SECONDS="${F3S_KEYS_WAIT_SECONDS:-10}"
+MOUNT_OPTIONS="${F3S_KEYS_MOUNT_OPTIONS:-ro}"
+LOGTAG="f3s-mount-keys"
+STRICT="NO"
+
+if [ "${1:-}" = "--strict" ]; then
+    STRICT="YES"
+fi
+
+log()
+{
+    logger -t "$LOGTAG" "$*"
+    echo "$LOGTAG: $*"
+}
+
+fail()
+{
+    log "$*"
+    if [ "$STRICT" = "YES" ]; then
+        exit 1
+    fi
+    exit 0
+}
+
+is_mounted()
+{
+    mount | grep -Eq " on ${MOUNTPOINT} \\("
+}
+
+find_device()
+{
+    for dev in $DEVICES; do
+        if [ -e "$dev" ]; then
+            echo "$dev"
+            return 0
+        fi
+    done
+    return 1
+}
+
+validate_keys()
+{
+    missing="$(
+        zfs get -H -t filesystem,volume -s local -o value,name keylocation 2>/dev/null |
+        while IFS="$(printf '\t')" read -r keylocation dataset; do
+            case "$keylocation" in
+                file://"$MOUNTPOINT"/*)
+                    keyfile="${keylocation#file://}"
+                    if [ ! -s "$keyfile" ] || [ ! -r "$keyfile" ]; then
+                        echo "${dataset}:${keyfile}"
+                    fi
+                    ;;
+            esac
+        done
+    )"
+
+    if [ -n "$missing" ]; then
+        echo "$missing" | while IFS=: read -r dataset keyfile; do
+            log "key file for ${dataset} is missing or unreadable: ${keyfile}"
+        done
+        return 1
+    fi
+
+    return 0
+}
+
+if is_mounted; then
+    log "$MOUNTPOINT is already mounted"
+    validate_keys || true
+    exit 0
+fi
+
+mkdir -p "$MOUNTPOINT" || fail "could not create $MOUNTPOINT"
+chmod 700 "$MOUNTPOINT" 2>/dev/null || true
+
+i=0
+dev=""
+while [ "$i" -le "$WAIT_SECONDS" ]; do
+    if dev="$(find_device)"; then
+        break
+    fi
+    sleep 1
+    i=$((i + 1))
+done
+
+if [ -z "$dev" ]; then
+    fail "no USB key device found; checked: $DEVICES"
+fi
+
+if ! fsck_ufs -p "$dev"; then
+    fail "fsck_ufs failed for $dev; leaving $MOUNTPOINT unmounted"
+fi
+
+if ! mount -t ufs -o "$MOUNT_OPTIONS" "$dev" "$MOUNTPOINT"; then
+    fail "could not mount $dev on $MOUNTPOINT"
+fi
+
+log "mounted $dev on $MOUNTPOINT with options $MOUNT_OPTIONS"
+validate_keys || true
+
+exit 0
diff --git a/f3s/freebsd-hosts/keys/f3skeys.rc b/f3s/freebsd-hosts/keys/f3skeys.rc
new file mode 100644
index 0000000..862fafc
--- /dev/null
+++ b/f3s/freebsd-hosts/keys/f3skeys.rc
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# PROVIDE: f3skeys
+# REQUIRE: zpool
+# BEFORE: zfskeys zfs zvol
+# KEYWORD: nojail
+
+. /etc/rc.subr
+
+name="f3skeys"
+desc="Mount f3s USB key stick before ZFS encrypted dataset keys are loaded"
+rcvar="f3skeys_enable"
+start_cmd="f3skeys_start"
+stop_cmd=":"
+
+: ${f3skeys_enable:="NO"}
+: ${f3skeys_flags:=""}
+
+f3skeys_start()
+{
+    if [ -x /usr/local/sbin/f3s-mount-keys ]; then
+        /usr/local/sbin/f3s-mount-keys $f3skeys_flags || true
+    else
+        logger -t f3skeys "/usr/local/sbin/f3s-mount-keys is missing"
+    fi
+}
+
+load_rc_config "$name"
+run_rc_command "$1"