diff options
| -rwxr-xr-x | scripts/test-install.sh | 263 | ||||
| -rw-r--r-- | tests/unit/test_test_install.bats | 152 |
2 files changed, 355 insertions, 60 deletions
diff --git a/scripts/test-install.sh b/scripts/test-install.sh index bb556cd..cbca9b3 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -26,6 +26,17 @@ VM_DISK_SIZE="20G" # holds 2222 (e.g. SSH_PORT=2223 scripts/test-install.sh single-disk). export SSH_PORT="${SSH_PORT:-2222}" export SSH_PASSWORD="archangel" + +# Wall-clock bound on a single remote command. ConnectTimeout only bounds the +# connection, so a guest that accepts SSH and then blocks in the kernel hangs +# the run: on 2026-08-03 a `zfs destroy` stuck behind an uninterruptible +# txg_quiesce sat for 40 minutes on a healthy connection and produced no +# results at all. Bounded, a wedged guest costs one scenario. +# +# The installer invocation in run_install is the sole legitimate long runner +# and raises this to INSTALL_TIMEOUT for its own call. Exported because +# run_install executes in a `bash -c` child that inherits only exported vars. +export SSH_CMD_TIMEOUT="${SSH_CMD_TIMEOUT:-120}" SERIAL_LOG="$LOG_DIR/serial.log" # Timeouts (seconds) @@ -33,7 +44,7 @@ BOOT_TIMEOUT=120 # INSTALL_TIMEOUT: 30 min. DKMS zfs compile + depmod on kernel 6.18+ in # a VM can exceed 10 min under host load. 600 was tight for 6.12; 1800 # gives headroom without masking real hangs. -INSTALL_TIMEOUT=1800 +export INSTALL_TIMEOUT=1800 SSH_TIMEOUT=30 VERIFY_TIMEOUT=60 @@ -89,6 +100,49 @@ list_configs() { done } +# Drop the cached archzfs repo before the first scenario. +# +# archzfs re-uploads its GitHub Releases assets under the same filenames, so +# pacoloco ends up holding package bodies that no longer match what its +# archzfs.db advertises. Every ZFS scenario then dies at pacstrap with +# "invalid or corrupted package" or "Maximum file size exceeded", which reads +# exactly like an installer regression and isn't one. It went stale twice on +# 2026-08-06, the second time *during* a run: fresh at scenario one, rotten by +# scenario six, and the whole ZFS half was lost. +# +# Clear the directory, not the zfs-dkms/zfs-utils globs build.sh uses. Removing +# the bodies while leaving a stale db just trades a checksum error for a size +# error, which is how the second failure disguised itself as a new bug. +# +# Best-effort by design. The cache is pacoloco-owned so removal needs root, and +# a run without root is still worth having — it just carries the risk this +# exists to remove, so say so rather than failing silently. +# +# The path is injectable so tests can exercise this against a temp directory. +# Without that it hardcoded a system path, and merely sourcing the file and +# calling the function ran `sudo rm -rf` on the real cache — which is what +# happened the first time this was written. +clear_archzfs_cache() { + local dir="${ARCHZFS_CACHE_DIR:-/var/cache/pacoloco/pkgs/archzfs}" + [[ -d "$dir" ]] || return 0 + + # This is `rm -rf` under sudo on an overridable path, so validate the shape + # before running it. A mistyped override should cost a warning, not the + # machine. + if [[ "$(basename "$dir")" != archzfs || "$(dirname "$dir")" == / ]]; then + warn "Refusing to clear implausible archzfs cache path: $dir" + return 0 + fi + + if sudo -n rm -rf "$dir" 2>/dev/null; then + info "Cleared cached archzfs repo (pacoloco refetches on first use)" + else + warn "Could not clear $dir — needs root." + warn "A stale archzfs cache fails every ZFS scenario at pacstrap; clear it by hand if that happens." + fi + return 0 +} + find_iso() { ISO_FILE=$(ls -t "$PROJECT_DIR/out/"*.iso 2>/dev/null | head -1) if [[ -z "$ISO_FILE" ]]; then @@ -391,11 +445,21 @@ send_luks_passphrase() { return 0 } -# Send ZFS passphrase via QEMU monitor sendkey -# Two passphrase prompts occur (both on VGA framebuffer, not serial): -# 1. ZFSBootMenu prompts to unlock the pool and show boot environments -# 2. mkinitcpio's zfs hook prompts again when the selected kernel boots -# We detect the UEFI firmware log to time the first, then wait for the second. +# Send ZFS passphrase via QEMU monitor sendkey. +# +# One prompt, not two. ZFSBootMenu asks in order to read the kernel and +# initramfs; the booted initramfs then loads the key from the keyfile +# configure_zfs_keyfile bakes into the image, silently. Before that fix the +# key did not survive kexec and the initramfs asked a second time, so this +# used to send twice on fixed sleeps. +# +# Sending a second time now would type the passphrase into whatever is on +# screen once the system is up, which is a login prompt. +# +# Both prompts render on the VGA framebuffer, never the serial console, so +# there is nothing to match on: the UEFI firmware's ZFSBootMenu handoff is +# the last thing serial shows, and the wait after it is necessarily a fixed +# sleep rather than a poll. send_zfs_passphrase() { local test_name="$1" local passphrase="$2" @@ -415,27 +479,111 @@ send_zfs_passphrase() { done info "ZFSBootMenu loading detected after ${waited}s" - # Prompt 1: ZFSBootMenu passphrase (unlocks pool to show boot menu) + # The only prompt: ZFSBootMenu unlocking the pool. step "Waiting for ZFSBootMenu passphrase prompt (15s)..." sleep 15 - step "Sending ZFS passphrase (1/2: ZFSBootMenu)..." + step "Sending ZFS passphrase..." monitor_sendkeys "$monitor_sock" "$passphrase" - info "ZFSBootMenu passphrase sent" + info "ZFS passphrase sent" - # Prompt 2: mkinitcpio zfs hook passphrase (re-imports pool during kernel boot) - step "Waiting for initramfs passphrase prompt (30s)..." - sleep 30 - step "Sending ZFS passphrase (2/2: initramfs)..." - monitor_sendkeys "$monitor_sock" "$passphrase" - info "Initramfs passphrase sent" + return 0 +} + +# Which passphrase-entry path a boot from disk needs: "luks", "zfs", or empty. +# NO_ENCRYPT wins over a passphrase being present, because the test configs set +# both — sending a passphrase to an unencrypted boot would type it at a login +# prompt. LUKS is checked first, preserving the precedence the inline block had. +config_encrypt_flag() { + local config="$1" + local luks_pass zfs_pass no_encrypt + luks_pass=$(grep '^LUKS_PASSPHRASE=' "$config" | cut -d= -f2) + zfs_pass=$(grep '^ZFS_PASSPHRASE=' "$config" | cut -d= -f2) + no_encrypt=$(grep '^NO_ENCRYPT=' "$config" | cut -d= -f2) + + [[ "$no_encrypt" == "yes" ]] && return 0 + if [[ -n "$luks_pass" ]]; then + echo "luks" + elif [[ -n "$zfs_pass" ]]; then + echo "zfs" + fi + return 0 +} + +# Start the guest from its installed disk and get it past any passphrase +# prompt. Returns 0 once the VM is running and unlocked, 1 otherwise. Callers +# own their own failure bookkeeping, which is why this doesn't touch +# TESTS_FAILED or cleanup_disks. +boot_from_disk() { + local config="$1" + local config_name disk_count encrypt_flag vm_pid + config_name=$(basename "$config" .conf) + disk_count=$(get_disk_count "$config") + encrypt_flag=$(config_encrypt_flag "$config") + + vm_pid=$(start_vm_from_disk "$config_name" "$disk_count" "$encrypt_flag") + if [[ -z "$vm_pid" ]]; then + error "Failed to start VM from disk" + return 1 + fi + info "VM started from disk (PID: $vm_pid)" + + case "$encrypt_flag" in + luks) + if ! send_luks_passphrase "$config_name" \ + "$(grep '^LUKS_PASSPHRASE=' "$config" | cut -d= -f2)" "$disk_count"; then + error "Failed to send LUKS passphrase" + return 1 + fi + ;; + zfs) + if ! send_zfs_passphrase "$config_name" \ + "$(grep '^ZFS_PASSPHRASE=' "$config" | cut -d= -f2)"; then + error "Failed to send ZFS passphrase" + return 1 + fi + ;; + esac + return 0 +} + +# Restart the guest and wait for SSH to come back. +# +# Needed after a live-root `zfs rollback`. Rolling back a mounted root leaves +# the running system inconsistent with its own filesystem, and the pool in a +# state where the next ZFS command can wedge: on 2026-08-03 the cleanup +# `zfs destroy` blocked in cv_wait_common behind an uninterruptible +# txg_quiesce, and the suite sat dead for 40 minutes. A reboot exports and +# reimports the pool, which clears that, and it is also the only honest way to +# observe rolled-back contents — the page and dentry caches otherwise keep +# serving the pre-rollback view. +# +# Assumes SSH is enabled, which holds because every caller sits inside +# run_test's SSH branch. +reboot_guest() { + local config="$1" + local config_name installed_password + config_name=$(basename "$config" .conf) + installed_password=$(grep '^ROOT_PASSWORD=' "$config" | cut -d= -f2) + + step "Rebooting guest..." + stop_vm "$config_name" true + : > "$SERIAL_LOG" + + boot_from_disk "$config" || return 1 + if ! wait_for_ssh "$BOOT_TIMEOUT" "$installed_password"; then + error "Guest did not return on SSH after reboot" + return 1 + fi + info "Guest back up after reboot" return 0 } # Run SSH command (uses SSH_PASSWORD by default, or INSTALLED_PASSWORD if set) ssh_cmd() { local password="${INSTALLED_PASSWORD:-$SSH_PASSWORD}" - sshpass -p "$password" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + timeout "$SSH_CMD_TIMEOUT" \ + sshpass -p "$password" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ -p "$SSH_PORT" root@localhost "$@" 2>/dev/null } @@ -519,7 +667,10 @@ run_install() { fi # Run the installer (NO_ENCRYPT is set in the config file, not via flag) - ssh_cmd "archangel --config-file /root/test.conf" || return 1 + # The one call that legitimately runs for many minutes. Raise the + # per-command bound to match the outer timeout wrapping run_install, so + # the short default can't kill a healthy install mid-pacstrap. + SSH_CMD_TIMEOUT="$INSTALL_TIMEOUT" ssh_cmd "archangel --config-file /root/test.conf" || return 1 return 0 } @@ -868,14 +1019,39 @@ verify_zfssnapshot_wrapper() { error "zfssnapshot rollback --name failed" return 1 fi + # Reboot before looking, and before touching ZFS again. + # + # That rollback just reverted the mounted root underneath the running OS. + # Two things follow. The running system can't be trusted to report what is + # on disk, because the page and dentry caches keep serving the pre-rollback + # view — that's what made this check fail on four scenarios and pass on two + # identical ones. And the pool is left fragile: the next ZFS command can + # wedge it, which is exactly what the cleanup destroy below did on + # 2026-08-03, blocking in cv_wait_common behind an uninterruptible + # txg_quiesce until the run was killed 40 minutes later. + # + # A reboot exports and reimports the pool, which resolves both. + if ! reboot_guest "$config"; then + error "Guest did not come back after the rollback reboot" + return 1 + fi + if ! ssh_cmd "test -f '$sentinel'"; then error "Rollback did not restore sentinel — wrapper rollback path broken" + # Distinguish the two failures for whoever reads this next: a rollback + # that never happened leaves the snapshot and no file; one that + # happened leaves a consistent dataset. Post-reboot both answers are + # trustworthy, which they were not before. + error " ls: $(ssh_cmd "ls -la '$sentinel' 2>&1" | head -1)" + error " rollback snapshot still listed: $(ssh_cmd "zfs list -t snapshot -H -o name 2>&1 | grep -c wrapper-rollback")" return 1 fi info "Round-trip rollback via wrapper restored sentinel" # Cleanup: destroy the wrapper-rollback snapshot we left behind so # the VM ends in a clean state matching how verify_rollback leaves it. + # Safe here and not before the reboot, because the pool has been + # reimported since the rollback. ssh_cmd "echo yes | zfssnapshot delete --name '$rb_snap_name' 2>&1" >/dev/null || true return 0 @@ -1035,54 +1211,17 @@ run_test() { step "Booting from installed disk..." : > "$SERIAL_LOG" # Clear serial log - # Determine encryption mode (needs monitor socket for passphrase entry via sendkey) - local luks_passphrase - luks_passphrase=$(grep '^LUKS_PASSPHRASE=' "$config" | cut -d= -f2) - local zfs_passphrase - zfs_passphrase=$(grep '^ZFS_PASSPHRASE=' "$config" | cut -d= -f2) - local no_encrypt - no_encrypt=$(grep '^NO_ENCRYPT=' "$config" | cut -d= -f2) - local encrypt_flag="" - if [[ -n "$luks_passphrase" && "$no_encrypt" != "yes" ]]; then - encrypt_flag="luks" - elif [[ -n "$zfs_passphrase" && "$no_encrypt" != "yes" ]]; then - encrypt_flag="zfs" - fi - - local vm_pid2 - vm_pid2=$(start_vm_from_disk "$config_name" "$disk_count" "$encrypt_flag") - - if [[ -z "$vm_pid2" ]]; then - error "Failed to start VM from disk" + # boot_from_disk owns the encryption-mode decision and the monitor sendkey + # passphrase entry. Same sequence reboot_guest uses mid-test, so the two + # paths can't drift. + if ! boot_from_disk "$config"; then + stop_vm "$config_name" + cp "$SERIAL_LOG" "$LOG_DIR/${config_name}-reboot-serial.log" 2>/dev/null || true cleanup_disks "$config_name" TESTS_FAILED=$((TESTS_FAILED + 1)) FAILED_TESTS+=("$config_name") return 1 fi - info "VM started from disk (PID: $vm_pid2)" - - # If encryption is enabled, send passphrase via monitor sendkey - if [[ "$encrypt_flag" == "luks" ]]; then - if ! send_luks_passphrase "$config_name" "$luks_passphrase" "$disk_count"; then - error "Failed to send LUKS passphrase" - stop_vm "$config_name" - cp "$SERIAL_LOG" "$LOG_DIR/${config_name}-reboot-serial.log" 2>/dev/null || true - cleanup_disks "$config_name" - TESTS_FAILED=$((TESTS_FAILED + 1)) - FAILED_TESTS+=("$config_name") - return 1 - fi - elif [[ "$encrypt_flag" == "zfs" ]]; then - if ! send_zfs_passphrase "$config_name" "$zfs_passphrase"; then - error "Failed to send ZFS passphrase" - stop_vm "$config_name" - cp "$SERIAL_LOG" "$LOG_DIR/${config_name}-reboot-serial.log" 2>/dev/null || true - cleanup_disks "$config_name" - TESTS_FAILED=$((TESTS_FAILED + 1)) - FAILED_TESTS+=("$config_name") - return 1 - fi - fi # Check if SSH is enabled in the config local enable_ssh @@ -1212,6 +1351,10 @@ main() { # Find ISO find_iso + # Before the first scenario, so a stale upstream re-upload can't fail every + # ZFS install with an error that looks like a regression. + clear_archzfs_cache + # Determine which configs to run if [[ ${#configs[@]} -eq 0 ]]; then # Run all configs diff --git a/tests/unit/test_test_install.bats b/tests/unit/test_test_install.bats index bf43dd8..52ea037 100644 --- a/tests/unit/test_test_install.bats +++ b/tests/unit/test_test_install.bats @@ -318,6 +318,10 @@ error: failed to commit transaction (invalid or corrupted package (checksum)) @test "ssh_cmd picks up a caller-scoped INSTALLED_PASSWORD" { # Proves local-instead-of-export still reaches ssh_cmd: bash's dynamic # scoping exposes a caller's local to the functions it calls. + # + # timeout is stubbed as a pass-through because the real one is an external + # binary: it would exec the real sshpass and never see these stubs. + timeout() { shift; "$@"; } sshpass() { echo "$2"; } ssh() { :; } caller_with_local() { @@ -329,6 +333,7 @@ error: failed to commit transaction (invalid or corrupted package (checksum)) } @test "a caller-scoped INSTALLED_PASSWORD does not leak past a failed return" { + timeout() { shift; "$@"; } sshpass() { echo "$2"; } ssh() { :; } SSH_PASSWORD="live-iso-password" @@ -350,3 +355,150 @@ error: failed to commit transaction (invalid or corrupted package (checksum)) grep -qE '^[[:space:]]*local INSTALLED_PASSWORD=' "$src" ! grep -qE '^[[:space:]]*export INSTALLED_PASSWORD' "$src" } + +############################# +# config_encrypt_flag +############################# +# Decides which passphrase-entry path a reboot needs. It's the one pure piece +# of the boot-from-disk sequence, which is otherwise qemu orchestration, so it +# carries the precedence rules the rest of that sequence depends on. + +mkcfg() { + local f + f=$(mktemp) + printf '%s\n' "$@" > "$f" + echo "$f" +} + +@test "config_encrypt_flag reports luks for a LUKS config" { + local f; f=$(mkcfg 'LUKS_PASSPHRASE=secret' 'DISKS=/dev/vda') + [ "$(config_encrypt_flag "$f")" = "luks" ] + rm -f "$f" +} + +@test "config_encrypt_flag reports zfs for a ZFS-passphrase config" { + local f; f=$(mkcfg 'ZFS_PASSPHRASE=secret' 'DISKS=/dev/vda') + [ "$(config_encrypt_flag "$f")" = "zfs" ] + rm -f "$f" +} + +@test "config_encrypt_flag prefers luks when a config carries both" { + # Preserves the precedence the inline block had: LUKS is checked first, + # and a config with both is a misconfiguration rather than a real mode. + local f; f=$(mkcfg 'LUKS_PASSPHRASE=a' 'ZFS_PASSPHRASE=b') + [ "$(config_encrypt_flag "$f")" = "luks" ] + rm -f "$f" +} + +@test "config_encrypt_flag reports nothing when NO_ENCRYPT overrides a passphrase" { + # The test configs set a passphrase *and* NO_ENCRYPT=yes; sending a + # passphrase to an unencrypted boot would type it at a login prompt. + # + # Asserted through `run` on purpose. A bare [ -z "$(...)" ] also passes + # when the function doesn't exist, so it can't fail for the reason the + # test exists — checking status too makes absence register as 127. + local f; f=$(mkcfg 'ZFS_PASSPHRASE=testpass' 'NO_ENCRYPT=yes') + run config_encrypt_flag "$f" + [ "$status" -eq 0 ] + [ -z "$output" ] + rm -f "$f" +} + +@test "config_encrypt_flag reports nothing for an unencrypted config" { + local f; f=$(mkcfg 'DISKS=/dev/vda' 'HOSTNAME=x') + run config_encrypt_flag "$f" + [ "$status" -eq 0 ] + [ -z "$output" ] + rm -f "$f" +} + +############################# +# ssh_cmd timeout bound +############################# +# ConnectTimeout bounds the connection, not execution. On 2026-08-03 a remote +# `zfs destroy` blocked behind an uninterruptible txg_quiesce with the +# connection healthy, and the suite sat dead for 40 minutes. A wedged guest +# should cost one scenario, not the run. + +@test "ssh_cmd bounds the remote command with the default timeout" { + timeout() { echo "bound=$1"; } + run ssh_cmd true + [[ "$output" == "bound=$SSH_CMD_TIMEOUT" ]] +} + +@test "ssh_cmd honors a per-call timeout override" { + # run_install's installer call legitimately runs for many minutes and + # raises this; every other call keeps the short default. + timeout() { echo "bound=$1"; } + SSH_CMD_TIMEOUT=1800 run ssh_cmd true + [[ "$output" == "bound=1800" ]] +} + +############################# +# clear_archzfs_cache +############################# +# Exercised against an injected directory, never the real one. An earlier +# version of this block hardcoded the system path, so running the unit suite +# invoked `sudo rm -rf /var/cache/pacoloco/pkgs/archzfs` on the live machine. +# A unit test must not reach outside its sandbox. + +@test "clear_archzfs_cache is a no-op when the cache directory is absent" { + ARCHZFS_CACHE_DIR="$BATS_TEST_TMPDIR/absent" run clear_archzfs_cache + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "clear_archzfs_cache removes an existing cache directory" { + local dir="$BATS_TEST_TMPDIR/archzfs" + mkdir -p "$dir" + touch "$dir/archzfs.db" "$dir/zfs-dkms-1-1-x86_64.pkg.tar.zst" + # Stub sudo so the test needs no privilege and stays in its sandbox. + sudo() { shift; "$@"; } + ARCHZFS_CACHE_DIR="$dir" run clear_archzfs_cache + [ "$status" -eq 0 ] + [ ! -d "$dir" ] +} + +@test "clear_archzfs_cache clears the db too, not just the package bodies" { + # Removing bodies while leaving a stale archzfs.db trades a checksum error + # for "Maximum file size exceeded" — same cause, new message. + local dir="$BATS_TEST_TMPDIR/db/archzfs" + mkdir -p "$dir" + touch "$dir/archzfs.db" + sudo() { shift; "$@"; } + ARCHZFS_CACHE_DIR="$dir" run clear_archzfs_cache + [ ! -e "$dir/archzfs.db" ] +} + +@test "clear_archzfs_cache warns and succeeds when removal is denied" { + local dir="$BATS_TEST_TMPDIR/denied/archzfs" + mkdir -p "$dir" + sudo() { return 1; } + ARCHZFS_CACHE_DIR="$dir" run clear_archzfs_cache + # Non-fatal: a run without root is still worth having, but must say so. + [ "$status" -eq 0 ] + [[ "$output" == *"needs root"* ]] + [ -d "$dir" ] +} + +@test "clear_archzfs_cache refuses a path that isn't shaped like the cache" { + # The removal runs `rm -rf` under sudo, so a mistyped override must cost a + # warning rather than the machine. + local dir="$BATS_TEST_TMPDIR/not-the-cache" + mkdir -p "$dir" + sudo() { echo "SUDO RAN"; } + ARCHZFS_CACHE_DIR="$dir" run clear_archzfs_cache + [ "$status" -eq 0 ] + [[ "$output" == *"Refusing"* ]] + [[ "$output" != *"SUDO RAN"* ]] + [ -d "$dir" ] +} + +@test "clear_archzfs_cache refuses a top-level directory" { + sudo() { echo "SUDO RAN"; } + ARCHZFS_CACHE_DIR="/archzfs" run clear_archzfs_cache + # /archzfs won't exist, so this exits on the -d guard; the point is that + # neither guard lets a root-level path reach the removal. + [ "$status" -eq 0 ] + [[ "$output" != *"SUDO RAN"* ]] +} |