From c9f06a21fb1daa74c9db341636a2c4b118bda1d8 Mon Sep 17 00:00:00 2001
From: Paul Buetow <paul@buetow.org>
Date: Sat, 20 Jun 2026 17:28:36 +0300
Subject: carp: rollback sink to last snapshot on BACKUP transition
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

After a CARP failover where f1 serves as MASTER, any NFS writes go to
the sink dataset (zdata/sink/f0/zdata/enc/nfsdata). When f1 returns to
BACKUP, zfs receive from f0 fails with "destination has been modified
since most recent snapshot" because the filesystem state is ahead of the
last received snapshot.

Fix: on BACKUP transition, roll the sink back to the last zrepl snapshot
before setting readonly=on. Writes during the MASTER window (health
checks, test writes) are intentionally discarded — f0 is authoritative
and replication resumes cleanly from the common snapshot.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 f3s/freebsd-hosts/carp/carpcontrol.sh | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/f3s/freebsd-hosts/carp/carpcontrol.sh b/f3s/freebsd-hosts/carp/carpcontrol.sh
index 6caa447..527699b 100644
--- a/f3s/freebsd-hosts/carp/carpcontrol.sh
+++ b/f3s/freebsd-hosts/carp/carpcontrol.sh
@@ -44,10 +44,21 @@ case "$2" in
         service nfsd stop >/dev/null 2>&1
         service mountd stop >/dev/null 2>&1
         service nfsuserd stop >/dev/null 2>&1
-        # Restore readonly=on on the sink dataset so that zrepl can resume
-        # writing incoming snapshot data from f0.
+        # Restore the sink dataset for zrepl replication from f0.
+        # Any writes made while MASTER must be rolled back to the last
+        # received snapshot; otherwise zfs receive fails with "destination
+        # has been modified since most recent snapshot". Data written
+        # during the MASTER window is transient (health checks, test writes)
+        # and is intentionally discarded here.
         if [ "$HOSTNAME" != 'f0.lan.buetow.org' ]; then
-            zfs set readonly=on zdata/sink/f0/zdata/enc/nfsdata
+            SINK="zdata/sink/f0/zdata/enc/nfsdata"
+            LAST_SNAP=$(zfs list -t snapshot -Ho name "$SINK" 2>/dev/null | tail -1)
+            if [ -n "$LAST_SNAP" ]; then
+                zfs rollback "$LAST_SNAP" && \
+                    logger "CARP BACKUP: rolled back $SINK to $LAST_SNAP" || \
+                    logger "CARP BACKUP: WARNING rollback of $SINK to $LAST_SNAP failed"
+            fi
+            zfs set readonly=on "$SINK"
         fi
         logger "CARP BACKUP: NFS and stunnel services stopped"
         ;;
-- 
cgit v1.2.3