qr0: timeout + per-destination isolation for rsync sync_dist

Wrap each rsync in sync_dist in run_with_timeout (new SYNC_TIMEOUT config,
default 300s) so a hung or unreachable mirror cannot block the whole sync,
matching every other external call. Make destinations isolated: under
set -euo pipefail a single failing destination used to abort the loop and
silently skip the rest. Now each destination runs under a localized set +e
(the project's refresh_splash idiom), results are collected per destination,
a clear pass/fail summary is logged, and sync returns non-zero if any
destination failed while still attempting all of them.

SYNC_TIMEOUT is plumbed like TAR_TIMEOUT: shuriken.default.conf,
apply_config_defaults, print_config, verbose config log, and positive-integer
validation in both validate_config (generate path) and validate_sync_config
(sync path). No CLI flag, matching TAR_TIMEOUT.

Tests: a sync where one destination fails still attempts the others and exits
non-zero with the summary; SYNC_TIMEOUT=0 is rejected as a positive integer.
Adds install_rsync_spy_failing_one helper and updates the print_config
expected blocks.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

1 files changed, 90 insertions, 0 deletions

diff --git a/tests/cli.sh b/tests/cli.sh
index 28aacbb..2280a7e 100755
--- a/tests/cli.sh
+++ b/tests/cli.sh
@@ -1494,6 +1494,7 @@ TARBALL_SUFFIX=.tar
 TAR_TIMEOUT=120
 TAR_OPTS=( -c )
 SYNC_DELETE=yes
+SYNC_TIMEOUT=300
 SYNC_DESTINATIONS=( )
 ORIGINAL_BASEPATH=''
 EOF
@@ -1545,6 +1546,7 @@ TARBALL_SUFFIX=.tar
 TAR_TIMEOUT=120
 TAR_OPTS=( -c )
 SYNC_DELETE=yes
+SYNC_TIMEOUT=300
 SYNC_DESTINATIONS=( )
 ORIGINAL_BASEPATH=''
 EOF
@@ -1689,6 +1691,7 @@ TARBALL_SUFFIX=.tar
 TAR_TIMEOUT=120
 TAR_OPTS=( -c )
 SYNC_DELETE=yes
+SYNC_TIMEOUT=300
 SYNC_DESTINATIONS=( )
 ORIGINAL_BASEPATH=''
 EOF
@@ -1735,6 +1738,7 @@ TARBALL_SUFFIX=.tar
 TAR_TIMEOUT=120
 TAR_OPTS=( -c )
 SYNC_DELETE=yes
+SYNC_TIMEOUT=300
 SYNC_DESTINATIONS=( )
 ORIGINAL_BASEPATH=''
 EOF
@@ -1808,6 +1812,7 @@ TARBALL_SUFFIX=.tar
 TAR_TIMEOUT=120
 TAR_OPTS=( -c )
 SYNC_DELETE=yes
+SYNC_TIMEOUT=300
 SYNC_DESTINATIONS=( )
 ORIGINAL_BASEPATH=''
 EOF
@@ -6155,6 +6160,85 @@ test_sync_array_destination_preserves_spaces() {
     test::teardown
 }
 
+test_sync_isolates_failing_destination_and_reports_failure() {
+    # Regression for qr0: a failed destination must NOT abort the loop. The first
+    # destination fails, but rsync must still be invoked for the second, and the
+    # overall --sync must exit non-zero with a clear pass/fail summary.
+    local config_file
+    local dist_dir
+    local fake_bin
+    local rsync_log
+    local rsync_output
+    local sync_output
+    local fail_dest='admin@one.example:/var/www/one/'
+    local ok_dest='admin@two.example:/var/www/two/'
+
+    test::setup
+    fake_bin="$TEST_TMPDIR/bin"
+    config_file="$TEST_TMPDIR/shuriken.conf"
+    dist_dir="$TEST_TMPDIR/dist"
+    rsync_log="$TEST_TMPDIR/rsync.log"
+
+    test::install_rsync_spy_failing_one "$fake_bin"
+    mkdir -p "$dist_dir"
+    printf 'generated\n' > "$dist_dir/index.html"
+    {
+        printf 'DIST_DIR=%q\n' "$dist_dir"
+        printf 'SYNC_DESTINATIONS=(\n'
+        printf '    %q\n' "$fail_dest"
+        printf '    %q\n' "$ok_dest"
+        printf ')\n'
+    } > "$config_file"
+
+    sync_output=$(
+        cd "$TEST_TMPDIR"
+        PATH="$fake_bin:$PATH" TEST_RSYNC_LOG="$rsync_log" \
+            TEST_RSYNC_FAIL_DEST="$fail_dest" \
+            test::capture_failure_output "$TEST_SHURIKEN" --sync
+    )
+
+    # Both destinations were attempted despite the first one failing.
+    rsync_output=$(<"$rsync_log")
+    test::assert_contains "arg3=$fail_dest" "$rsync_output"
+    test::assert_contains "arg3=$ok_dest" "$rsync_output"
+    # The summary reports the successful and failed mirrors.
+    test::assert_contains "Sync succeeded for: $ok_dest" "$sync_output"
+    test::assert_contains "Sync failed for: $fail_dest" "$sync_output"
+    test::teardown
+}
+
+test_sync_timeout_rejects_non_positive_integer() {
+    # SYNC_TIMEOUT is validated as a positive integer like TAR_TIMEOUT.
+    local config_file
+    local dist_dir
+    local fake_bin
+    local output
+
+    test::setup
+    fake_bin="$TEST_TMPDIR/bin"
+    config_file="$TEST_TMPDIR/shuriken.conf"
+    dist_dir="$TEST_TMPDIR/dist"
+
+    test::install_rsync_spy "$fake_bin"
+    mkdir -p "$dist_dir"
+    printf 'generated\n' > "$dist_dir/index.html"
+    {
+        printf 'DIST_DIR=%q\n' "$dist_dir"
+        printf 'SYNC_TIMEOUT=%q\n' '0'
+        printf 'SYNC_DESTINATIONS=(%q)\n' 'admin@one.example:/var/www/one/'
+    } > "$config_file"
+
+    output=$(
+        cd "$TEST_TMPDIR"
+        PATH="$fake_bin:$PATH" TEST_RSYNC_LOG="$TEST_TMPDIR/rsync.log" \
+            test::capture_failure_output "$TEST_SHURIKEN" --sync
+    )
+
+    test::assert_contains 'SYNC_TIMEOUT' "$output"
+    test::assert_path_absent "$TEST_TMPDIR/rsync.log"
+    test::teardown
+}
+
 test_positional_commands_fail_without_deprecation() {
     local output
     local old_command
@@ -7158,6 +7242,12 @@ main() {
         '--sync array destination preserves spaces' \
         test_sync_array_destination_preserves_spaces
     test::run_case \
+        '--sync isolates a failing destination and reports failure' \
+        test_sync_isolates_failing_destination_and_reports_failure
+    test::run_case \
+        '--sync rejects non-positive SYNC_TIMEOUT' \
+        test_sync_timeout_rejects_non_positive_integer
+    test::run_case \
         'positional commands fail without deprecation output' \
         test_positional_commands_fail_without_deprecation
     test::run_case \