fix(test): reboot the guest after a live-root rollback

verify_zfssnapshot_wrapper rolled back zroot/ROOT/default while it was mounted and running, then immediately read the result and destroyed the snapshot. ZFS doesn't support rolling back a mounted root, and both follow-ups were unsafe. The page and dentry caches kept serving the pre-rollback view, so the sentinel check failed on four scenarios and passed on two identical ones. The cleanup destroy was worse: it blocked in cv_wait_common behind an uninterruptible txg_quiesce and wedged the pool, taking a whole run with it.

The check reboots the guest first now. A reboot exports and reimports the pool, which clears the fragile state and makes the on-disk contents readable. I pulled boot_from_disk and config_encrypt_flag out of run_test's inline boot block, so the mid-test and end-of-install paths share one implementation.

ssh_cmd is bounded by SSH_CMD_TIMEOUT, default 120s. ConnectTimeout only bounds the connection, so that wedged guest held a healthy session open and hung the run for 40 minutes. The installer call raises the bound to INSTALL_TIMEOUT, since it legitimately runs for many minutes.

send_zfs_passphrase sends once. Embedding the pool key left a single prompt, so the second send was typing the passphrase at a login prompt.

2 files changed, 239 insertions, 60 deletions

diff --git a/scripts/test-install.sh b/scripts/test-install.sh
index bb556cd..d61f95d 100755
--- a/scripts/test-install.sh
+++ b/scripts/test-install.sh
@@ -26,6 +26,17 @@ VM_DISK_SIZE="20G"
 # holds 2222 (e.g. SSH_PORT=2223 scripts/test-install.sh single-disk).
 export SSH_PORT="${SSH_PORT:-2222}"
 export SSH_PASSWORD="archangel"
+
+# Wall-clock bound on a single remote command. ConnectTimeout only bounds the
+# connection, so a guest that accepts SSH and then blocks in the kernel hangs
+# the run: on 2026-08-03 a `zfs destroy` stuck behind an uninterruptible
+# txg_quiesce sat for 40 minutes on a healthy connection and produced no
+# results at all. Bounded, a wedged guest costs one scenario.
+#
+# The installer invocation in run_install is the sole legitimate long runner
+# and raises this to INSTALL_TIMEOUT for its own call. Exported because
+# run_install executes in a `bash -c` child that inherits only exported vars.
+export SSH_CMD_TIMEOUT="${SSH_CMD_TIMEOUT:-120}"
 SERIAL_LOG="$LOG_DIR/serial.log"
 
 # Timeouts (seconds)
@@ -33,7 +44,7 @@ BOOT_TIMEOUT=120
 # INSTALL_TIMEOUT: 30 min. DKMS zfs compile + depmod on kernel 6.18+ in
 # a VM can exceed 10 min under host load. 600 was tight for 6.12; 1800
 # gives headroom without masking real hangs.
-INSTALL_TIMEOUT=1800
+export INSTALL_TIMEOUT=1800
 SSH_TIMEOUT=30
 VERIFY_TIMEOUT=60
 
@@ -391,11 +402,21 @@ send_luks_passphrase() {
     return 0
 }
 
-# Send ZFS passphrase via QEMU monitor sendkey
-# Two passphrase prompts occur (both on VGA framebuffer, not serial):
-#   1. ZFSBootMenu prompts to unlock the pool and show boot environments
-#   2. mkinitcpio's zfs hook prompts again when the selected kernel boots
-# We detect the UEFI firmware log to time the first, then wait for the second.
+# Send ZFS passphrase via QEMU monitor sendkey.
+#
+# One prompt, not two. ZFSBootMenu asks in order to read the kernel and
+# initramfs; the booted initramfs then loads the key from the keyfile
+# configure_zfs_keyfile bakes into the image, silently. Before that fix the
+# key did not survive kexec and the initramfs asked a second time, so this
+# used to send twice on fixed sleeps.
+#
+# Sending a second time now would type the passphrase into whatever is on
+# screen once the system is up, which is a login prompt.
+#
+# Both prompts render on the VGA framebuffer, never the serial console, so
+# there is nothing to match on: the UEFI firmware's ZFSBootMenu handoff is
+# the last thing serial shows, and the wait after it is necessarily a fixed
+# sleep rather than a poll.
 send_zfs_passphrase() {
     local test_name="$1"
     local passphrase="$2"
@@ -415,27 +436,111 @@ send_zfs_passphrase() {
     done
     info "ZFSBootMenu loading detected after ${waited}s"
 
-    # Prompt 1: ZFSBootMenu passphrase (unlocks pool to show boot menu)
+    # The only prompt: ZFSBootMenu unlocking the pool.
     step "Waiting for ZFSBootMenu passphrase prompt (15s)..."
     sleep 15
-    step "Sending ZFS passphrase (1/2: ZFSBootMenu)..."
+    step "Sending ZFS passphrase..."
     monitor_sendkeys "$monitor_sock" "$passphrase"
-    info "ZFSBootMenu passphrase sent"
+    info "ZFS passphrase sent"
 
-    # Prompt 2: mkinitcpio zfs hook passphrase (re-imports pool during kernel boot)
-    step "Waiting for initramfs passphrase prompt (30s)..."
-    sleep 30
-    step "Sending ZFS passphrase (2/2: initramfs)..."
-    monitor_sendkeys "$monitor_sock" "$passphrase"
-    info "Initramfs passphrase sent"
+    return 0
+}
+
+# Which passphrase-entry path a boot from disk needs: "luks", "zfs", or empty.
+# NO_ENCRYPT wins over a passphrase being present, because the test configs set
+# both — sending a passphrase to an unencrypted boot would type it at a login
+# prompt. LUKS is checked first, preserving the precedence the inline block had.
+config_encrypt_flag() {
+    local config="$1"
+    local luks_pass zfs_pass no_encrypt
+    luks_pass=$(grep '^LUKS_PASSPHRASE=' "$config" | cut -d= -f2)
+    zfs_pass=$(grep '^ZFS_PASSPHRASE=' "$config" | cut -d= -f2)
+    no_encrypt=$(grep '^NO_ENCRYPT=' "$config" | cut -d= -f2)
+
+    [[ "$no_encrypt" == "yes" ]] && return 0
+    if [[ -n "$luks_pass" ]]; then
+        echo "luks"
+    elif [[ -n "$zfs_pass" ]]; then
+        echo "zfs"
+    fi
+    return 0
+}
+
+# Start the guest from its installed disk and get it past any passphrase
+# prompt. Returns 0 once the VM is running and unlocked, 1 otherwise. Callers
+# own their own failure bookkeeping, which is why this doesn't touch
+# TESTS_FAILED or cleanup_disks.
+boot_from_disk() {
+    local config="$1"
+    local config_name disk_count encrypt_flag vm_pid
+    config_name=$(basename "$config" .conf)
+    disk_count=$(get_disk_count "$config")
+    encrypt_flag=$(config_encrypt_flag "$config")
+
+    vm_pid=$(start_vm_from_disk "$config_name" "$disk_count" "$encrypt_flag")
+    if [[ -z "$vm_pid" ]]; then
+        error "Failed to start VM from disk"
+        return 1
+    fi
+    info "VM started from disk (PID: $vm_pid)"
+
+    case "$encrypt_flag" in
+        luks)
+            if ! send_luks_passphrase "$config_name" \
+                    "$(grep '^LUKS_PASSPHRASE=' "$config" | cut -d= -f2)" "$disk_count"; then
+                error "Failed to send LUKS passphrase"
+                return 1
+            fi
+            ;;
+        zfs)
+            if ! send_zfs_passphrase "$config_name" \
+                    "$(grep '^ZFS_PASSPHRASE=' "$config" | cut -d= -f2)"; then
+                error "Failed to send ZFS passphrase"
+                return 1
+            fi
+            ;;
+    esac
+    return 0
+}
+
+# Restart the guest and wait for SSH to come back.
+#
+# Needed after a live-root `zfs rollback`. Rolling back a mounted root leaves
+# the running system inconsistent with its own filesystem, and the pool in a
+# state where the next ZFS command can wedge: on 2026-08-03 the cleanup
+# `zfs destroy` blocked in cv_wait_common behind an uninterruptible
+# txg_quiesce, and the suite sat dead for 40 minutes. A reboot exports and
+# reimports the pool, which clears that, and it is also the only honest way to
+# observe rolled-back contents — the page and dentry caches otherwise keep
+# serving the pre-rollback view.
+#
+# Assumes SSH is enabled, which holds because every caller sits inside
+# run_test's SSH branch.
+reboot_guest() {
+    local config="$1"
+    local config_name installed_password
+    config_name=$(basename "$config" .conf)
+    installed_password=$(grep '^ROOT_PASSWORD=' "$config" | cut -d= -f2)
+
+    step "Rebooting guest..."
+    stop_vm "$config_name" true
+    : > "$SERIAL_LOG"
 
+    boot_from_disk "$config" || return 1
+
+    if ! wait_for_ssh "$BOOT_TIMEOUT" "$installed_password"; then
+        error "Guest did not return on SSH after reboot"
+        return 1
+    fi
+    info "Guest back up after reboot"
     return 0
 }
 
 # Run SSH command (uses SSH_PASSWORD by default, or INSTALLED_PASSWORD if set)
 ssh_cmd() {
     local password="${INSTALLED_PASSWORD:-$SSH_PASSWORD}"
-    sshpass -p "$password" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+    timeout "$SSH_CMD_TIMEOUT" \
+        sshpass -p "$password" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
         -p "$SSH_PORT" root@localhost "$@" 2>/dev/null
 }
 
@@ -519,7 +624,10 @@ run_install() {
     fi
 
     # Run the installer (NO_ENCRYPT is set in the config file, not via flag)
-    ssh_cmd "archangel --config-file /root/test.conf" || return 1
+    # The one call that legitimately runs for many minutes. Raise the
+    # per-command bound to match the outer timeout wrapping run_install, so
+    # the short default can't kill a healthy install mid-pacstrap.
+    SSH_CMD_TIMEOUT="$INSTALL_TIMEOUT" ssh_cmd "archangel --config-file /root/test.conf" || return 1
 
     return 0
 }
@@ -868,14 +976,39 @@ verify_zfssnapshot_wrapper() {
         error "zfssnapshot rollback --name failed"
         return 1
     fi
+    # Reboot before looking, and before touching ZFS again.
+    #
+    # That rollback just reverted the mounted root underneath the running OS.
+    # Two things follow. The running system can't be trusted to report what is
+    # on disk, because the page and dentry caches keep serving the pre-rollback
+    # view — that's what made this check fail on four scenarios and pass on two
+    # identical ones. And the pool is left fragile: the next ZFS command can
+    # wedge it, which is exactly what the cleanup destroy below did on
+    # 2026-08-03, blocking in cv_wait_common behind an uninterruptible
+    # txg_quiesce until the run was killed 40 minutes later.
+    #
+    # A reboot exports and reimports the pool, which resolves both.
+    if ! reboot_guest "$config"; then
+        error "Guest did not come back after the rollback reboot"
+        return 1
+    fi
+
     if ! ssh_cmd "test -f '$sentinel'"; then
         error "Rollback did not restore sentinel — wrapper rollback path broken"
+        # Distinguish the two failures for whoever reads this next: a rollback
+        # that never happened leaves the snapshot and no file; one that
+        # happened leaves a consistent dataset. Post-reboot both answers are
+        # trustworthy, which they were not before.
+        error "  ls: $(ssh_cmd "ls -la '$sentinel' 2>&1" | head -1)"
+        error "  rollback snapshot still listed: $(ssh_cmd "zfs list -t snapshot -H -o name 2>&1 | grep -c wrapper-rollback")"
         return 1
     fi
     info "Round-trip rollback via wrapper restored sentinel"
 
     # Cleanup: destroy the wrapper-rollback snapshot we left behind so
     # the VM ends in a clean state matching how verify_rollback leaves it.
+    # Safe here and not before the reboot, because the pool has been
+    # reimported since the rollback.
     ssh_cmd "echo yes | zfssnapshot delete --name '$rb_snap_name' 2>&1" >/dev/null || true
 
     return 0
@@ -1035,54 +1168,17 @@ run_test() {
     step "Booting from installed disk..."
     : > "$SERIAL_LOG"  # Clear serial log
 
-    # Determine encryption mode (needs monitor socket for passphrase entry via sendkey)
-    local luks_passphrase
-    luks_passphrase=$(grep '^LUKS_PASSPHRASE=' "$config" | cut -d= -f2)
-    local zfs_passphrase
-    zfs_passphrase=$(grep '^ZFS_PASSPHRASE=' "$config" | cut -d= -f2)
-    local no_encrypt
-    no_encrypt=$(grep '^NO_ENCRYPT=' "$config" | cut -d= -f2)
-    local encrypt_flag=""
-    if [[ -n "$luks_passphrase" && "$no_encrypt" != "yes" ]]; then
-        encrypt_flag="luks"
-    elif [[ -n "$zfs_passphrase" && "$no_encrypt" != "yes" ]]; then
-        encrypt_flag="zfs"
-    fi
-
-    local vm_pid2
-    vm_pid2=$(start_vm_from_disk "$config_name" "$disk_count" "$encrypt_flag")
-
-    if [[ -z "$vm_pid2" ]]; then
-        error "Failed to start VM from disk"
+    # boot_from_disk owns the encryption-mode decision and the monitor sendkey
+    # passphrase entry. Same sequence reboot_guest uses mid-test, so the two
+    # paths can't drift.
+    if ! boot_from_disk "$config"; then
+        stop_vm "$config_name"
+        cp "$SERIAL_LOG" "$LOG_DIR/${config_name}-reboot-serial.log" 2>/dev/null || true
         cleanup_disks "$config_name"
         TESTS_FAILED=$((TESTS_FAILED + 1))
         FAILED_TESTS+=("$config_name")
         return 1
     fi
-    info "VM started from disk (PID: $vm_pid2)"
-
-    # If encryption is enabled, send passphrase via monitor sendkey
-    if [[ "$encrypt_flag" == "luks" ]]; then
-        if ! send_luks_passphrase "$config_name" "$luks_passphrase" "$disk_count"; then
-            error "Failed to send LUKS passphrase"
-            stop_vm "$config_name"
-            cp "$SERIAL_LOG" "$LOG_DIR/${config_name}-reboot-serial.log" 2>/dev/null || true
-            cleanup_disks "$config_name"
-            TESTS_FAILED=$((TESTS_FAILED + 1))
-            FAILED_TESTS+=("$config_name")
-            return 1
-        fi
-    elif [[ "$encrypt_flag" == "zfs" ]]; then
-        if ! send_zfs_passphrase "$config_name" "$zfs_passphrase"; then
-            error "Failed to send ZFS passphrase"
-            stop_vm "$config_name"
-            cp "$SERIAL_LOG" "$LOG_DIR/${config_name}-reboot-serial.log" 2>/dev/null || true
-            cleanup_disks "$config_name"
-            TESTS_FAILED=$((TESTS_FAILED + 1))
-            FAILED_TESTS+=("$config_name")
-            return 1
-        fi
-    fi
 
     # Check if SSH is enabled in the config
     local enable_ssh
diff --git a/tests/unit/test_test_install.bats b/tests/unit/test_test_install.bats
index bf43dd8..34da2b0 100644
--- a/tests/unit/test_test_install.bats
+++ b/tests/unit/test_test_install.bats
@@ -318,6 +318,10 @@ error: failed to commit transaction (invalid or corrupted package (checksum))
 @test "ssh_cmd picks up a caller-scoped INSTALLED_PASSWORD" {
     # Proves local-instead-of-export still reaches ssh_cmd: bash's dynamic
     # scoping exposes a caller's local to the functions it calls.
+    #
+    # timeout is stubbed as a pass-through because the real one is an external
+    # binary: it would exec the real sshpass and never see these stubs.
+    timeout() { shift; "$@"; }
     sshpass() { echo "$2"; }
     ssh() { :; }
     caller_with_local() {
@@ -329,6 +333,7 @@ error: failed to commit transaction (invalid or corrupted package (checksum))
 }
 
 @test "a caller-scoped INSTALLED_PASSWORD does not leak past a failed return" {
+    timeout() { shift; "$@"; }
     sshpass() { echo "$2"; }
     ssh() { :; }
     SSH_PASSWORD="live-iso-password"
@@ -350,3 +355,81 @@ error: failed to commit transaction (invalid or corrupted package (checksum))
     grep -qE '^[[:space:]]*local INSTALLED_PASSWORD=' "$src"
     ! grep -qE '^[[:space:]]*export INSTALLED_PASSWORD' "$src"
 }
+
+#############################
+# config_encrypt_flag
+#############################
+# Decides which passphrase-entry path a reboot needs. It's the one pure piece
+# of the boot-from-disk sequence, which is otherwise qemu orchestration, so it
+# carries the precedence rules the rest of that sequence depends on.
+
+mkcfg() {
+    local f
+    f=$(mktemp)
+    printf '%s\n' "$@" > "$f"
+    echo "$f"
+}
+
+@test "config_encrypt_flag reports luks for a LUKS config" {
+    local f; f=$(mkcfg 'LUKS_PASSPHRASE=secret' 'DISKS=/dev/vda')
+    [ "$(config_encrypt_flag "$f")" = "luks" ]
+    rm -f "$f"
+}
+
+@test "config_encrypt_flag reports zfs for a ZFS-passphrase config" {
+    local f; f=$(mkcfg 'ZFS_PASSPHRASE=secret' 'DISKS=/dev/vda')
+    [ "$(config_encrypt_flag "$f")" = "zfs" ]
+    rm -f "$f"
+}
+
+@test "config_encrypt_flag prefers luks when a config carries both" {
+    # Preserves the precedence the inline block had: LUKS is checked first,
+    # and a config with both is a misconfiguration rather than a real mode.
+    local f; f=$(mkcfg 'LUKS_PASSPHRASE=a' 'ZFS_PASSPHRASE=b')
+    [ "$(config_encrypt_flag "$f")" = "luks" ]
+    rm -f "$f"
+}
+
+@test "config_encrypt_flag reports nothing when NO_ENCRYPT overrides a passphrase" {
+    # The test configs set a passphrase *and* NO_ENCRYPT=yes; sending a
+    # passphrase to an unencrypted boot would type it at a login prompt.
+    #
+    # Asserted through `run` on purpose. A bare [ -z "$(...)" ] also passes
+    # when the function doesn't exist, so it can't fail for the reason the
+    # test exists — checking status too makes absence register as 127.
+    local f; f=$(mkcfg 'ZFS_PASSPHRASE=testpass' 'NO_ENCRYPT=yes')
+    run config_encrypt_flag "$f"
+    [ "$status" -eq 0 ]
+    [ -z "$output" ]
+    rm -f "$f"
+}
+
+@test "config_encrypt_flag reports nothing for an unencrypted config" {
+    local f; f=$(mkcfg 'DISKS=/dev/vda' 'HOSTNAME=x')
+    run config_encrypt_flag "$f"
+    [ "$status" -eq 0 ]
+    [ -z "$output" ]
+    rm -f "$f"
+}
+
+#############################
+# ssh_cmd timeout bound
+#############################
+# ConnectTimeout bounds the connection, not execution. On 2026-08-03 a remote
+# `zfs destroy` blocked behind an uninterruptible txg_quiesce with the
+# connection healthy, and the suite sat dead for 40 minutes. A wedged guest
+# should cost one scenario, not the run.
+
+@test "ssh_cmd bounds the remote command with the default timeout" {
+    timeout() { echo "bound=$1"; }
+    run ssh_cmd true
+    [[ "$output" == "bound=$SSH_CMD_TIMEOUT" ]]
+}
+
+@test "ssh_cmd honors a per-call timeout override" {
+    # run_install's installer call legitimately runs for many minutes and
+    # raises this; every other call keeps the short default.
+    timeout() { echo "bound=$1"; }
+    SSH_CMD_TIMEOUT=1800 run ssh_cmd true
+    [[ "$output" == "bound=1800" ]]
+}