Fix r-node NFS shutdown ordering

Amp-Thread-ID: https://ampcode.com/threads/T-019fc2f1-f70e-74aa-8efe-650e1a1c3097
Co-authored-by: Amp <amp@ampcode.com>

6 files changed, 78 insertions, 2 deletions

diff --git a/f3s/r-nodes/Rexfile b/f3s/r-nodes/Rexfile
index fd61326..e05dc5d 100644
--- a/f3s/r-nodes/Rexfile
+++ b/f3s/r-nodes/Rexfile
@@ -2,6 +2,7 @@
 #
 # Run from repository root:
 #   rex -f f3s/r-nodes/Rexfile nfs_mount_monitor
+#   rex -f f3s/r-nodes/Rexfile persistent_journal
 #
 # All tasks connect as root (r-nodes require root for systemd and
 # /usr/local/bin writes; paul user has no sudo configured on these VMs).
@@ -41,6 +42,8 @@ parallelism 3;
 #   /etc/default/nfs-mount-monitor          (tunable: NFS_FAIL_THRESHOLD)
 #   /etc/systemd/system/nfs-mount-monitor.service
 #   /etc/systemd/system/nfs-mount-monitor.timer
+#   /etc/systemd/system/data-nfs-k3svolumes.mount.d/10-stunnel-ordering.conf
+#   /etc/systemd/system/k3s.service.d/10-nfs-ordering.conf
 #   /var/lib/nfs-mount-monitor/             (state dir for fail-count file)
 #
 # Idempotent: Rex only writes the file when content changes; the
@@ -115,6 +118,34 @@ task 'nfs_mount_monitor',
       mode      => '644',
       on_change => sub { $changed = 1 };
 
+    # Keep the local TLS transport running until the NFS mount is detached.
+    file '/etc/systemd/system/data-nfs-k3svolumes.mount.d',
+      ensure => 'directory',
+      owner  => 'root',
+      group  => 'root',
+      mode   => '755';
+
+    file '/etc/systemd/system/data-nfs-k3svolumes.mount.d/10-stunnel-ordering.conf',
+      source    => catfile( $monitor_dir, 'nfs-stunnel-ordering.conf' ),
+      owner     => 'root',
+      group     => 'root',
+      mode      => '644',
+      on_change => sub { $changed = 1 };
+
+    # Stop k3s before detaching its host-path NFS filesystem.
+    file '/etc/systemd/system/k3s.service.d',
+      ensure => 'directory',
+      owner  => 'root',
+      group  => 'root',
+      mode   => '755';
+
+    file '/etc/systemd/system/k3s.service.d/10-nfs-ordering.conf',
+      source    => catfile( $monitor_dir, 'k3s-nfs-ordering.conf' ),
+      owner     => 'root',
+      group     => 'root',
+      mode      => '644',
+      on_change => sub { $changed = 1 };
+
     if ($changed) {
         Rex::Logger::info('Files changed — reloading systemd and restarting timer');
         run 'systemctl daemon-reload';
@@ -126,6 +157,43 @@ task 'nfs_mount_monitor',
     run 'systemctl enable nfs-mount-monitor.timer';
   };
 
+# Keep the previous boot's journal so failed shutdowns can be diagnosed after
+# a forced guest stop or reboot. Limit its disk use on these small VMs.
+desc 'Enable persistent systemd journal storage on r0/r1/r2';
+task 'persistent_journal',
+  group => 'r_nodes',
+  sub {
+    my $changed = 0;
+
+    file '/etc/systemd/journald.conf.d',
+      ensure => 'directory',
+      owner  => 'root',
+      group  => 'root',
+      mode   => '755';
+
+    file '/etc/systemd/journald.conf.d/10-persistent.conf',
+      source    => catfile( $RNODES_DIR, 'journald-persistent.conf' ),
+      owner     => 'root',
+      group     => 'root',
+      mode      => '644',
+      on_change => sub { $changed = 1 };
+
+    file '/var/log/journal',
+      ensure => 'directory',
+      owner  => 'root',
+      group  => 'systemd-journal',
+      mode   => '2755';
+
+    if ($changed) {
+        run 'systemd-tmpfiles --create --prefix /var/log/journal';
+        run 'systemctl restart systemd-journald';
+    }
+
+    # Also converges after the journal directory was recreated independently
+    # of the already-correct drop-in file.
+    run 'journalctl --flush';
+  };
+
 1;
 
 # vim: syntax=perl
diff --git a/f3s/r-nodes/journald-persistent.conf b/f3s/r-nodes/journald-persistent.conf
new file mode 100644
index 0000000..d37d6ee
--- /dev/null
+++ b/f3s/r-nodes/journald-persistent.conf
@@ -0,0 +1,3 @@
+[Journal]
+Storage=persistent
+SystemMaxUse=256M
diff --git a/f3s/r-nodes/nfs-mount-monitor/k3s-nfs-ordering.conf b/f3s/r-nodes/nfs-mount-monitor/k3s-nfs-ordering.conf
new file mode 100644
index 0000000..339643d
--- /dev/null
+++ b/f3s/r-nodes/nfs-mount-monitor/k3s-nfs-ordering.conf
@@ -0,0 +1,2 @@
+[Unit]
+After=data-nfs-k3svolumes.mount
diff --git a/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.service b/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.service
index e31dbe1..ee224eb 100644
--- a/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.service
+++ b/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.service
@@ -9,7 +9,7 @@
 
 [Unit]
 Description=NFS Mount Health Monitor
-After=network-online.target
+After=network-online.target remote-fs.target
 
 [Service]
 Type=oneshot
diff --git a/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.timer b/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.timer
index 31251f2..4ccd17f 100644
--- a/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.timer
+++ b/f3s/r-nodes/nfs-mount-monitor/nfs-mount-monitor.timer
@@ -7,7 +7,7 @@
 
 [Unit]
 Description=Run NFS Mount Health Monitor every 10 seconds
-Requires=nfs-mount-monitor.service
+After=remote-fs.target
 
 [Timer]
 OnBootSec=30s
diff --git a/f3s/r-nodes/nfs-mount-monitor/nfs-stunnel-ordering.conf b/f3s/r-nodes/nfs-mount-monitor/nfs-stunnel-ordering.conf
new file mode 100644
index 0000000..f1de94a
--- /dev/null
+++ b/f3s/r-nodes/nfs-mount-monitor/nfs-stunnel-ordering.conf
@@ -0,0 +1,3 @@
+[Unit]
+Requires=stunnel.service
+After=stunnel.service