test(maint): add VM and nspawn remedy scenario harness

Nine break/fix/assert scenarios run the real maint fix inside the test VM. The runner batches non-conflicting scenarios into one VM boot, restores the snapshot only between groups, and leaves the base image pristine afterwards. A systemd-nspawn fast lane runs the pacman-level group against a cached pacstrap rootfs in seconds. The plan layer validates the scenario contract without a VM and carries a 19-test unit suite. make test-maint wires the VM lane in.

The zfs lane is filtered but unexercised: the FS_PROFILE=zfs base image fails to build (ZFS DKMS module not found on linux-lts 6.18). The failure is tracked in the todo.

11 files changed, 851 insertions, 0 deletions

diff --git a/scripts/testing/maint-scenarios/10-journal-vacuum.sh b/scripts/testing/maint-scenarios/10-journal-vacuum.sh
new file mode 100644
index 0000000..b61de49
--- /dev/null
+++ b/scripts/testing/maint-scenarios/10-journal-vacuum.sh
@@ -0,0 +1,30 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: the journal vacuum executes against live journald at the TOML floor.
+#
+# What this proves: the TOML edit reaches the remedy's argv (vacuum target
+# and warn threshold are the same key), journalctl --vacuum-size runs for
+# real against live journald, and the metric grades ok against the edited
+# floor afterwards.
+#
+# What it deliberately does NOT prove: size reduction. The key is int-GB and
+# bottoms out at 1, vacuum only touches archived files, and a VM-sized
+# journal never nears 1 GB — a genuine reduction test would need journald
+# rate-limit surgery plus ~1 GB of spam per run. A pre/post usage comparison
+# is also unsound here: journald keeps landing in-flight writes (the spam is
+# async, and every ssh hop logs), so usage can grow across a correct vacuum.
+SCENARIO_DESC="journal vacuum runs at the TOML size floor"
+SCENARIO_GROUP="logs"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "sed -i 's/^journal_disk_warn_gb *=.*/journal_disk_warn_gb = 1/' /root/.config/archsetup/maintenance-thresholds.toml" \
+        && mexec "for i in \$(seq 1 20); do head -c 900000 /dev/urandom | base64 | systemd-cat -t maint-scenario-spam; done; journalctl --rotate; journalctl --flush >/dev/null 2>&1 || true"
+}
+
+scenario_fix() {
+    mfix journal_vacuum
+}
+
+scenario_assert() {
+    massert_metric journal_disk ok
+}
diff --git a/scripts/testing/maint-scenarios/11-coredump-age-out.sh b/scripts/testing/maint-scenarios/11-coredump-age-out.sh
new file mode 100644
index 0000000..06d6fa0
--- /dev/null
+++ b/scripts/testing/maint-scenarios/11-coredump-age-out.sh
@@ -0,0 +1,21 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: coredump files beyond the forensic window are aged out.
+#
+# coredumpctl has no clean verb, so the remedy is a find -mtime +N -delete
+# over /var/lib/systemd/coredump. Break plants one file older than the
+# window and one fresh file; assert checks exactly the old one is gone.
+SCENARIO_DESC="aged coredump files removed, fresh ones kept"
+SCENARIO_GROUP="logs"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "mkdir -p /var/lib/systemd/coredump && touch -d '30 days ago' '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && touch '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'"
+}
+
+scenario_fix() {
+    mfix coredump_clean
+}
+
+scenario_assert() {
+    mexec "test ! -e '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && test -e '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'"
+}
diff --git a/scripts/testing/maint-scenarios/20-cache-keep3.sh b/scripts/testing/maint-scenarios/20-cache-keep3.sh
new file mode 100644
index 0000000..a9a6ee5
--- /dev/null
+++ b/scripts/testing/maint-scenarios/20-cache-keep3.sh
@@ -0,0 +1,24 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: paccache -rk3 trims an installed package's cache to 3 versions.
+#
+# paccache decides from filenames, so five fake versions of a package that
+# IS installed (bash) stand in for a fat cache. The bootstrap may leave a
+# real bash package in the cache too, so the assert uses the ordering
+# invariant rather than a count: the two newest fakes survive any keep-3
+# window that contains them, and the two oldest fall outside it whether or
+# not a real (newer) version is cached alongside.
+SCENARIO_DESC="pacman cache trimmed to three versions per package"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "for v in 1.0.1 1.0.2 1.0.3 1.0.4 1.0.5; do touch /var/cache/pacman/pkg/bash-\$v-1-x86_64.pkg.tar.zst; done; ls /var/cache/pacman/pkg/ | grep -c '^bash-'"
+}
+
+scenario_fix() {
+    mfix cache_clean
+}
+
+scenario_assert() {
+    mexec "test -e /var/cache/pacman/pkg/bash-1.0.5-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-1.0.4-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.2-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.1-1-x86_64.pkg.tar.zst"
+}
diff --git a/scripts/testing/maint-scenarios/21-cache-uninstalled.sh b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh
new file mode 100644
index 0000000..ea788e8
--- /dev/null
+++ b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh
@@ -0,0 +1,20 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: paccache -ruk0 drops every cached version of uninstalled packages.
+#
+# Fake versions of a package that is NOT installed disappear entirely, while
+# an installed package's cached version survives the uninstalled-only pass.
+SCENARIO_DESC="cached versions of uninstalled packages dropped"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "touch /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst"
+}
+
+scenario_fix() {
+    mfix cache_clean_uninstalled
+}
+
+scenario_assert() {
+    mexec "test ! -e /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst"
+}
diff --git a/scripts/testing/maint-scenarios/22-orphan-remove.sh b/scripts/testing/maint-scenarios/22-orphan-remove.sh
new file mode 100644
index 0000000..8412128
--- /dev/null
+++ b/scripts/testing/maint-scenarios/22-orphan-remove.sh
@@ -0,0 +1,22 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: a real orphan is created, detected, and removed by name.
+#
+# Installing `tree` --asdeps with no dependents makes it a true orphan
+# (pacman -Qtdq lists it). The remedy is item-based: maint fix orphan_remove
+# tree. Assert covers both the package state and the metric verdict.
+SCENARIO_DESC="orphan package removed via the item-based remedy"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "pacman -S --noconfirm --asdeps tree && pacman -Qtdq | grep -qx tree"
+}
+
+scenario_fix() {
+    mfix orphan_remove tree
+}
+
+scenario_assert() {
+    mexec "! pacman -Qi tree >/dev/null 2>&1 && ! pacman -Qtdq | grep -qx tree" \
+        && massert_metric pkg_orphans ok
+}
diff --git a/scripts/testing/maint-scenarios/23-pacnew-delete.sh b/scripts/testing/maint-scenarios/23-pacnew-delete.sh
new file mode 100644
index 0000000..23cd8d1
--- /dev/null
+++ b/scripts/testing/maint-scenarios/23-pacnew-delete.sh
@@ -0,0 +1,22 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: a .pacnew file is deleted through the validated priv verb.
+#
+# pacdiff --output finds the planted file; the remedy takes the path as its
+# item and rm's it. A neighbouring non-pacnew file proves the verb's path
+# validation keeps the blast radius to the named file.
+SCENARIO_DESC="planted .pacnew removed by path"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "cp /etc/pacman.conf /etc/pacman.conf.pacnew && pacdiff --output | grep -q pacman.conf.pacnew"
+}
+
+scenario_fix() {
+    mfix pacnew_delete /etc/pacman.conf.pacnew
+}
+
+scenario_assert() {
+    mexec "test ! -e /etc/pacman.conf.pacnew && test -e /etc/pacman.conf" \
+        && massert_metric pkg_pacnew ok
+}
diff --git a/scripts/testing/maint-scenarios/30-unit-reset.sh b/scripts/testing/maint-scenarios/30-unit-reset.sh
new file mode 100644
index 0000000..07eed53
--- /dev/null
+++ b/scripts/testing/maint-scenarios/30-unit-reset.sh
@@ -0,0 +1,28 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: a failed unit's state is reset.
+#
+# A transient oneshot that exits 1 lands in systemctl --failed; the remedy
+# resets it and the failed list returns to its pre-break size. The unit name
+# rides the remedy's item argument, exercising the priv verb's unit-name
+# validation. The base image carries its own failed unit (grub-btrfsd — no
+# snapshot dirs in the VM), so the assert scopes to this scenario's unit and
+# the count delta, never the absolute failed list.
+SCENARIO_DESC="failed unit reset clears the failed list"
+SCENARIO_GROUP="systemd"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    PRE_FAILED_COUNT=$(mexec "systemctl --failed --no-legend | wc -l") \
+        && mexec "systemd-run --unit=maint-scenario-fail --service-type=oneshot --no-block /bin/false; sleep 2; systemctl is-failed maint-scenario-fail.service"
+}
+
+scenario_fix() {
+    mfix unit_reset maint-scenario-fail.service
+}
+
+scenario_assert() {
+    local pre="$PRE_FAILED_COUNT"
+    unset PRE_FAILED_COUNT  # don't leak scenario state past this scenario
+    mexec "! systemctl is-failed maint-scenario-fail.service >/dev/null 2>&1" \
+        && [ "$(mexec "systemctl --failed --no-legend | wc -l")" = "$pre" ]
+}
diff --git a/scripts/testing/maint-scenarios/31-fstrim-enable.sh b/scripts/testing/maint-scenarios/31-fstrim-enable.sh
new file mode 100644
index 0000000..2d548a1
--- /dev/null
+++ b/scripts/testing/maint-scenarios/31-fstrim-enable.sh
@@ -0,0 +1,22 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: fstrim.timer enabled and started from the disabled default.
+#
+# The base image ships util-linux with fstrim.timer disabled — the exact
+# state the metric flags. The remedy is systemctl enable --now.
+SCENARIO_DESC="fstrim.timer enabled and started"
+SCENARIO_GROUP="systemd"
+SCENARIO_PROFILES="btrfs"
+
+scenario_break() {
+    # Default state, but pin it so the scenario stays honest if a future
+    # base image enables the timer.
+    mexec "systemctl disable --now fstrim.timer >/dev/null 2>&1 || true; ! systemctl is-enabled fstrim.timer >/dev/null 2>&1"
+}
+
+scenario_fix() {
+    mfix fstrim_enable
+}
+
+scenario_assert() {
+    mexec "systemctl is-enabled fstrim.timer && systemctl is-active fstrim.timer"
+}
diff --git a/scripts/testing/maint-scenarios/32-cron-enable.sh b/scripts/testing/maint-scenarios/32-cron-enable.sh
new file mode 100644
index 0000000..39d4de6
--- /dev/null
+++ b/scripts/testing/maint-scenarios/32-cron-enable.sh
@@ -0,0 +1,20 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: cronie enabled and started (the stopped-cron posture).
+#
+# Bootstrap installs cronie but never enables it — the broken state is the
+# package's default. The remedy enables + starts the service.
+SCENARIO_DESC="cronie enabled and started from the stopped state"
+SCENARIO_GROUP="systemd"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "systemctl disable --now cronie.service >/dev/null 2>&1 || true; ! systemctl is-active cronie.service >/dev/null 2>&1"
+}
+
+scenario_fix() {
+    mfix cron_enable
+}
+
+scenario_assert() {
+    mexec "systemctl is-enabled cronie.service && systemctl is-active cronie.service"
+}
diff --git a/scripts/testing/run-maint-nspawn.sh b/scripts/testing/run-maint-nspawn.sh
new file mode 100644
index 0000000..04d4a32
--- /dev/null
+++ b/scripts/testing/run-maint-nspawn.sh
@@ -0,0 +1,268 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Run pacman-level maint remedy scenarios in a systemd-nspawn container
+# Author: Craig Jennings <craigmartinjennings@gmail.com>
+# License: GNU GPLv3
+#
+# The fast lane for the maint scenario suite: pure pacman-level cases (the
+# "packages" scenario group — cache trims, orphan removal, pacnew deletion)
+# don't need a booted machine, so they run in seconds against a pacstrap'd
+# rootfs instead of minutes against the VM. Everything else (systemd, logs)
+# needs pid 1 and journald — that stays in run-maint-scenarios.sh.
+#
+# The scenario files are shared with the VM runner and see the same helpers
+# (mexec/mmaint/mfix/massert_metric); only the transport differs — the
+# helper trio is deliberately duplicated between the two runners rather
+# than abstracted over ssh-vs-nspawn.
+#
+# The rootfs is cached at $NSPAWN_ROOT and rebuilt with --fresh. Building
+# and running need root (pacstrap, systemd-nspawn) via sudo.
+#
+# Usage: run-maint-nspawn.sh [--list] [--fresh]
+#   --list   print the validated scenario plan and exit (no root needed)
+#   --fresh  delete the cached rootfs and pacstrap a new one
+# Env: MAINT_SCENARIO_DIR   scenario dir (default: maint-scenarios/)
+#      MAINT_SRC            maint package source tree
+#                           (default: ~/.dotfiles/maint/src/maint)
+#      NSPAWN_ROOT          rootfs dir (default: /var/tmp/archsetup-maint-nspawn)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+source "$SCRIPT_DIR/lib/logging.sh"
+
+SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}"
+MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}"
+THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml"
+NSPAWN_ROOT="${NSPAWN_ROOT:-/var/tmp/archsetup-maint-nspawn}"
+NSPAWN_GROUP="packages"
+
+LIST_ONLY=false
+FRESH=false
+
+usage() {
+    echo "Usage: $0 [--list] [--fresh]"
+    echo "  --list   print the validated scenario plan and exit"
+    echo "  --fresh  rebuild the cached rootfs from scratch"
+    echo "Env: MAINT_SCENARIO_DIR, MAINT_SRC, NSPAWN_ROOT"
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --list)  LIST_ONLY=true; shift ;;
+        --fresh) FRESH=true; shift ;;
+        *) usage; exit 1 ;;
+    esac
+done
+
+# ─── Plan: the pacman-level subset ───────────────────────────────────
+# Same contract validation as the VM runner; the selection rule is the
+# group name — "packages" is pacman-level by construction.
+
+S_FILES=() S_NAMES=() S_DESCS=()
+
+_scenario_var() {  # <file> <varname>
+    bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \
+        _probe "$1" "$2" 2>/dev/null
+}
+
+_validate_scenario() {  # <file> -> fatal on contract violation
+    local f="$1" base probe_err
+    base=$(basename "$f")
+    probe_err=$(bash -c 'set -eu; source "$1"
+        : "${SCENARIO_DESC:?missing SCENARIO_DESC}"
+        : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}"
+        : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}"
+        for p in $SCENARIO_PROFILES; do
+            case "$p" in btrfs|zfs|any) ;; *)
+                echo "bad profile token: $p" >&2; exit 1 ;;
+            esac
+        done
+        declare -f scenario_break scenario_fix scenario_assert >/dev/null \
+            || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \
+        _probe "$f" 2>&1 >/dev/null) \
+        || fatal "scenario contract violation in $base: $probe_err"
+}
+
+build_plan() {
+    [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR"
+    local f base
+    for f in "$SCENARIO_DIR"/*.sh; do
+        [ -e "$f" ] || break
+        _validate_scenario "$f"
+        [ "$(_scenario_var "$f" SCENARIO_GROUP)" = "$NSPAWN_GROUP" ] || continue
+        base=$(basename "$f" .sh)
+        S_FILES+=("$f")
+        S_NAMES+=("${base#[0-9][0-9]-}")
+        S_DESCS+=("$(_scenario_var "$f" SCENARIO_DESC)")
+    done
+    [ ${#S_FILES[@]} -gt 0 ] \
+        || fatal "no '$NSPAWN_GROUP'-group scenarios in $SCENARIO_DIR"
+}
+
+print_plan() {
+    local i
+    echo "maint nspawn scenario plan (group: $NSPAWN_GROUP)"
+    for i in "${!S_FILES[@]}"; do
+        echo "  ${S_NAMES[$i]} — ${S_DESCS[$i]}"
+    done
+}
+
+if [ "$LIST_ONLY" = "true" ]; then
+    LOGFILE=/dev/null
+    build_plan
+    print_plan
+    exit 0
+fi
+
+# ─── Run ─────────────────────────────────────────────────────────────
+
+TIMESTAMP=$(date +'%Y%m%d-%H%M%S')
+RESULTS_DIR="$PROJECT_ROOT/test-results/maint-nspawn-$TIMESTAMP"
+mkdir -p "$RESULTS_DIR"
+LOGFILE="$RESULTS_DIR/scenarios.log"
+init_logging "$LOGFILE"
+
+build_plan
+
+section "Maint Remedy Scenarios (nspawn): $TIMESTAMP"
+info "Rootfs: $NSPAWN_ROOT"
+info "Maint source: $MAINT_SRC"
+print_plan | tee -a "$LOGFILE"
+
+command -v pacstrap >/dev/null \
+    || fatal "pacstrap not found — pacman -S arch-install-scripts"
+command -v systemd-nspawn >/dev/null || fatal "systemd-nspawn not found"
+[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC"
+[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML"
+sudo -n true 2>/dev/null || fatal "needs passwordless sudo (pacstrap/nspawn)"
+
+# ─── Target helpers (available to scenario scripts) ──────────────────
+
+mexec() {  # run a command in the container as root
+    sudo systemd-nspawn -q -D "$NSPAWN_ROOT" --pipe \
+        /bin/bash -c "$*" 2>> "$LOGFILE"
+}
+
+mmaint() {
+    mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*"
+}
+
+mfix() {
+    local id="$1"; shift || true
+    step "maint fix $id ${*:+$* }--dry-run"
+    mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \
+        || { error "dry-run failed: maint fix $id $*"; return 1; }
+    step "maint fix $id $*"
+    mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE"
+    return "${PIPESTATUS[0]}"
+}
+
+massert_metric() {  # <metric-id> <expected-severity>
+    local mid="$1" want="$2" got
+    got=$(mmaint status --json | python3 -c "
+import json, sys
+env = json.load(sys.stdin)
+ms = [m for m in env['metrics'] if m['id'] == '$mid']
+print(ms[0]['severity'] if ms else 'MISSING')
+")
+    if [ "$got" = "$want" ]; then
+        success "metric $mid is $want"
+        return 0
+    fi
+    error "metric $mid: expected $want, got $got"
+    return 1
+}
+
+# ─── Rootfs ──────────────────────────────────────────────────────────
+
+if [ "$FRESH" = "true" ] && [ -d "$NSPAWN_ROOT" ]; then
+    step "Removing cached rootfs (--fresh)"
+    sudo rm -rf "$NSPAWN_ROOT"
+fi
+
+if [ ! -x "$NSPAWN_ROOT/usr/bin/python3" ]; then
+    section "Building rootfs (pacstrap)"
+    sudo mkdir -p "$NSPAWN_ROOT"
+    # shellcheck disable=SC2024  # the log is user-owned by design
+    sudo pacstrap -c -K "$NSPAWN_ROOT" base python pacman-contrib expac \
+        >> "$LOGFILE" 2>&1 || fatal "pacstrap failed (see $LOGFILE)"
+    success "Rootfs built"
+else
+    info "Reusing cached rootfs (--fresh to rebuild)"
+fi
+
+section "Installing maint tree + thresholds into the container"
+sudo rm -rf "$NSPAWN_ROOT/root/maint-pkg"
+sudo mkdir -p "$NSPAWN_ROOT/root/maint-pkg" "$NSPAWN_ROOT/root/.config/archsetup"
+sudo cp -r "$MAINT_SRC" "$NSPAWN_ROOT/root/maint-pkg/"
+sudo find "$NSPAWN_ROOT/root/maint-pkg" -name '__pycache__' -type d \
+    -exec rm -rf {} + 2>/dev/null || true
+sudo cp "$THRESHOLDS_TOML" \
+    "$NSPAWN_ROOT/root/.config/archsetup/maintenance-thresholds.toml"
+
+step "Smoke: maint status --json runs in the container"
+mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \
+    || fatal "maint status failed in the container"
+success "Container ready"
+
+# ─── Execution ───────────────────────────────────────────────────────
+
+PASS=() FAIL=()
+
+run_scenario() {  # <index>
+    local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}"
+    section "Scenario: $name — ${S_DESCS[$1]}"
+    # shellcheck disable=SC1090
+    source "$f"
+    local ok=true
+    step "break"
+    if ! scenario_break; then
+        error "$name: break step failed"
+        ok=false
+    fi
+    if [ "$ok" = "true" ]; then
+        step "fix"
+        if ! scenario_fix; then
+            error "$name: fix step failed"
+            ok=false
+        fi
+    fi
+    if [ "$ok" = "true" ]; then
+        step "assert"
+        if ! scenario_assert; then
+            error "$name: post-state assertion failed"
+            ok=false
+        fi
+    fi
+    unset -f scenario_break scenario_fix scenario_assert
+    unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES
+    if [ "$ok" = "true" ]; then
+        success "PASS: $name"
+        PASS+=("$name")
+    else
+        error "FAIL: $name"
+        FAIL+=("$name")
+    fi
+}
+
+for i in "${!S_FILES[@]}"; do
+    run_scenario "$i"
+done
+
+# ─── Report ──────────────────────────────────────────────────────────
+
+section "Results"
+for n in "${PASS[@]}"; do success "PASS $n"; done
+for n in "${FAIL[@]}"; do error "FAIL $n"; done
+info "Log: $LOGFILE"
+info "Rootfs kept for reuse: $NSPAWN_ROOT (--fresh to rebuild)"
+
+if [ ${#FAIL[@]} -gt 0 ]; then
+    error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed"
+    exit 1
+fi
+success "All ${#PASS[@]} scenarios passed"
+exit 0
diff --git a/scripts/testing/run-maint-scenarios.sh b/scripts/testing/run-maint-scenarios.sh
new file mode 100644
index 0000000..448f6d9
--- /dev/null
+++ b/scripts/testing/run-maint-scenarios.sh
@@ -0,0 +1,374 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Run maintenance-console remedy scenarios in the test VM
+# Author: Craig Jennings <craigmartinjennings@gmail.com>
+# License: GNU GPLv3
+#
+# Layer 3 of the maint test strategy: each scenario breaks the VM in a known
+# way over ssh, runs the real `maint fix <id>` inside the VM, and asserts the
+# post-state. Scenarios live in maint-scenarios/*.sh and declare a GROUP;
+# scenarios in one group are non-conflicting and share a VM boot, and a
+# stop -> restore -> boot cycle runs only between groups (snapshot ops need
+# the VM stopped). A scenario that fails may leave residue behind for its
+# in-group successors -- the group restore boundary contains it, so read a
+# multi-failure group front to back.
+#
+# Boot plan:
+#   1. restore clean-install, boot, bootstrap (packages + maint tree + TOML)
+#   2. stop, snapshot maint-ready
+#   3. per group: boot, run scenarios, stop, restore maint-ready
+#   4. restore clean-install, delete maint-ready (base stays pristine)
+#
+# Usage: run-maint-scenarios.sh [--list] [--group NAME] [--keep]
+#   --list   print the validated batch plan and exit (no VM, no KVM needed)
+#   --group  run (or list) only the named group
+#   --keep   keep the VM running in its post-run state; skip the restores
+# Env: FS_PROFILE=btrfs|zfs        base image + scenario profile filter
+#      MAINT_SCENARIO_DIR          scenario dir (default: maint-scenarios/)
+#      MAINT_SRC                   maint package source tree
+#                                  (default: ~/.dotfiles/maint/src/maint)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+source "$SCRIPT_DIR/lib/logging.sh"
+source "$SCRIPT_DIR/lib/vm-utils.sh"
+
+SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}"
+MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}"
+THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml"
+ROOT_PASSWORD="archsetup"
+READY_SNAPSHOT="maint-ready"
+
+LIST_ONLY=false
+ONLY_GROUP=""
+KEEP_VM=false
+
+usage() {
+    echo "Usage: $0 [--list] [--group NAME] [--keep]"
+    echo "  --list   print the validated scenario plan and exit (no VM)"
+    echo "  --group  run only the named scenario group"
+    echo "  --keep   keep the VM running after the run (skip restores)"
+    echo "Env: FS_PROFILE=btrfs|zfs, MAINT_SCENARIO_DIR, MAINT_SRC"
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --list)  LIST_ONLY=true; shift ;;
+        --group) ONLY_GROUP="${2:?--group requires a value}"; shift 2 ;;
+        --keep)  KEEP_VM=true; shift ;;
+        *) usage; exit 1 ;;
+    esac
+done
+
+# ─── Plan: enumerate, validate, group ────────────────────────────────
+# The plan layer is pure -- no VM, no KVM -- so --list works anywhere and
+# the unit suite (tests/maint-scenarios/) exercises it directly.
+
+# Parallel arrays over scenarios, in file order.
+S_FILES=() S_NAMES=() S_DESCS=() S_GROUPS=()
+# Groups in first-appearance order.
+GROUP_ORDER=()
+
+# Extract one declared var from a scenario file by sourcing it in a clean
+# subshell. Scenario files must only define vars + functions; any top-level
+# command fails here (no helpers exist), which is the contract.
+_scenario_var() {  # <file> <varname>
+    bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \
+        _probe "$1" "$2" 2>/dev/null
+}
+
+_validate_scenario() {  # <file> -> fatal on contract violation
+    local f="$1" base probe_err
+    base=$(basename "$f")
+    probe_err=$(bash -c 'set -eu; source "$1"
+        : "${SCENARIO_DESC:?missing SCENARIO_DESC}"
+        : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}"
+        : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}"
+        for p in $SCENARIO_PROFILES; do
+            case "$p" in btrfs|zfs|any) ;; *)
+                echo "bad profile token: $p" >&2; exit 1 ;;
+            esac
+        done
+        declare -f scenario_break scenario_fix scenario_assert >/dev/null \
+            || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \
+        _probe "$f" 2>&1 >/dev/null) \
+        || fatal "scenario contract violation in $base: $probe_err"
+}
+
+_profile_matches() {  # <profiles> -> 0 if scenario runs under $FS_PROFILE
+    local p
+    for p in $1; do
+        [ "$p" = "any" ] || [ "$p" = "$FS_PROFILE" ] && return 0
+    done
+    return 1
+}
+
+build_plan() {
+    [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR"
+    local f base name desc group profiles seen_groups=" "
+    local any=false
+    for f in "$SCENARIO_DIR"/*.sh; do
+        [ -e "$f" ] || break
+        any=true
+        _validate_scenario "$f"
+        base=$(basename "$f" .sh)
+        name="${base#[0-9][0-9]-}"
+        desc=$(_scenario_var "$f" SCENARIO_DESC)
+        group=$(_scenario_var "$f" SCENARIO_GROUP)
+        profiles=$(_scenario_var "$f" SCENARIO_PROFILES)
+        _profile_matches "$profiles" || continue
+        [ -n "$ONLY_GROUP" ] && [ "$group" != "$ONLY_GROUP" ] && continue
+        S_FILES+=("$f"); S_NAMES+=("$name")
+        S_DESCS+=("$desc"); S_GROUPS+=("$group")
+        case "$seen_groups" in
+            *" $group "*) ;;
+            *) GROUP_ORDER+=("$group"); seen_groups="$seen_groups$group " ;;
+        esac
+    done
+    [ "$any" = "true" ] || fatal "no scenario files in $SCENARIO_DIR"
+    if [ -n "$ONLY_GROUP" ] && [ ${#S_FILES[@]} -eq 0 ]; then
+        fatal "no scenarios match group '$ONLY_GROUP' (profile $FS_PROFILE)"
+    fi
+    [ ${#S_FILES[@]} -gt 0 ] \
+        || fatal "no scenarios match profile $FS_PROFILE"
+}
+
+print_plan() {
+    local g i
+    echo "maint scenario plan (profile: $FS_PROFILE)"
+    for g in "${GROUP_ORDER[@]}"; do
+        echo "group $g:"
+        for i in "${!S_FILES[@]}"; do
+            [ "${S_GROUPS[$i]}" = "$g" ] \
+                && echo "  ${S_NAMES[$i]} — ${S_DESCS[$i]}"
+        done
+    done
+}
+
+# --list needs no logging file, VM, or KVM. Give fatal() a LOGFILE target
+# only when we actually run.
+if [ "$LIST_ONLY" = "true" ]; then
+    LOGFILE=/dev/null
+    build_plan
+    print_plan
+    exit 0
+fi
+
+# ─── Run ─────────────────────────────────────────────────────────────
+
+TIMESTAMP=$(date +'%Y%m%d-%H%M%S')
+RESULTS_DIR="$PROJECT_ROOT/test-results/maint-$TIMESTAMP"
+mkdir -p "$RESULTS_DIR"
+LOGFILE="$RESULTS_DIR/scenarios.log"
+init_logging "$LOGFILE"
+init_vm_paths "$PROJECT_ROOT/vm-images"
+
+build_plan
+
+section "Maint Remedy Scenarios: $TIMESTAMP"
+info "Profile: $FS_PROFILE (image: $(basename "$DISK_PATH"))"
+info "Maint source: $MAINT_SRC"
+print_plan | tee -a "$LOGFILE"
+
+[ -f "$DISK_PATH" ] || fatal "base disk not found: $DISK_PATH — build it: FS_PROFILE=$FS_PROFILE $SCRIPT_DIR/create-base-vm.sh"
+snapshot_exists "$DISK_PATH" "clean-install" \
+    || fatal "snapshot 'clean-install' not found on $DISK_PATH"
+[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC"
+[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML"
+check_prerequisites || fatal "missing prerequisites"
+
+CLEANUP_DONE=0
+cleanup_scenarios() {
+    [ "$CLEANUP_DONE" = "1" ] && return 0
+    CLEANUP_DONE=1
+    [ "$KEEP_VM" = "true" ] && return 0
+    # Never silent: a failed restore/delete here leaves the base image dirty
+    # for the next `make test`, so name it even though the trap can't abort.
+    stop_qemu 2>/dev/null \
+        || echo "[!] cleanup: stop_qemu failed — VM may still hold the image" >&2
+    restore_snapshot "$DISK_PATH" "clean-install" 2>/dev/null \
+        || echo "[!] cleanup: clean-install restore FAILED — base image is dirty" >&2
+    if snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT"; then
+        delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" 2>/dev/null \
+            || echo "[!] cleanup: stray '$READY_SNAPSHOT' snapshot left on the base image" >&2
+    fi
+}
+trap cleanup_scenarios EXIT
+
+# ─── Target helpers (available to scenario scripts) ──────────────────
+
+mexec() {  # run a command in the VM as root
+    vm_exec "$ROOT_PASSWORD" "$@"
+}
+
+# maint invocation inside the VM. Root runs the verbs directly, so
+# MAINT_SUDO is empty (same switch the unit suites use).
+mmaint() {
+    mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*"
+}
+
+# Run one remedy: dry-run first (argv parity gate — the printed argv is the
+# arm-press string), then for real. Fails if either exits non-zero.
+mfix() {
+    local id="$1"; shift || true
+    step "maint fix $id ${*:+$* }--dry-run"
+    mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \
+        || { error "dry-run failed: maint fix $id $*"; return 1; }
+    step "maint fix $id $*"
+    mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE"
+    return "${PIPESTATUS[0]}"
+}
+
+# Assert a metric's severity in `maint status --json`.
+massert_metric() {  # <metric-id> <expected-severity>
+    local mid="$1" want="$2" got
+    got=$(mmaint status --json | python3 -c "
+import json, sys
+env = json.load(sys.stdin)
+ms = [m for m in env['metrics'] if m['id'] == '$mid']
+print(ms[0]['severity'] if ms else 'MISSING')
+")
+    if [ "$got" = "$want" ]; then
+        success "metric $mid is $want"
+        return 0
+    fi
+    error "metric $mid: expected $want, got $got"
+    return 1
+}
+
+# ─── Bootstrap ───────────────────────────────────────────────────────
+
+boot_vm() {
+    start_qemu "$DISK_PATH" "disk" "" "none" || fatal "failed to start VM"
+    wait_for_ssh "$ROOT_PASSWORD" 180 || fatal "VM SSH not available"
+}
+
+bootstrap_vm() {
+    section "Bootstrap: packages + maint tree + thresholds"
+    step "Refreshing keyring + installing runtime deps"
+    # The base image can be months old: refresh the keyring first so the
+    # dep install doesn't die on expired signatures.
+    mexec "pacman -Sy --noconfirm archlinux-keyring" >> "$LOGFILE" 2>&1 \
+        || fatal "keyring refresh failed (network up?)"
+    mexec "pacman -S --noconfirm python pacman-contrib expac cronie" \
+        >> "$LOGFILE" 2>&1 || fatal "dep install failed"
+
+    step "Copying maint package tree"
+    local tarball
+    tarball=$(mktemp)
+    tar -C "$(dirname "$MAINT_SRC")" --exclude='__pycache__' -czf "$tarball" \
+        "$(basename "$MAINT_SRC")"
+    copy_to_vm "$tarball" "/tmp/maint-src.tgz" "$ROOT_PASSWORD"
+    rm -f "$tarball"
+    mexec "rm -rf /root/maint-pkg && mkdir -p /root/maint-pkg && tar -C /root/maint-pkg -xzf /tmp/maint-src.tgz && rm /tmp/maint-src.tgz" \
+        >> "$LOGFILE" 2>&1 || fatal "maint tree extract failed"
+
+    step "Installing thresholds TOML"
+    copy_to_vm "$THRESHOLDS_TOML" "/tmp/maintenance-thresholds.toml" \
+        "$ROOT_PASSWORD"
+    mexec "mkdir -p /root/.config/archsetup && mv /tmp/maintenance-thresholds.toml /root/.config/archsetup/maintenance-thresholds.toml" \
+        >> "$LOGFILE" 2>&1 || fatal "TOML install failed"
+
+    step "Smoke: maint status --json runs"
+    mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \
+        || fatal "maint status failed in the VM"
+    success "Bootstrap complete ($(python3 -c "
+import json; print(len(json.load(open('$RESULTS_DIR/bootstrap-status.json'))['metrics']))") metrics probed)"
+}
+
+# ─── Execution ───────────────────────────────────────────────────────
+
+PASS=() FAIL=()
+
+run_scenario() {  # <index>
+    local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}"
+    section "Scenario: $name — ${S_DESCS[$1]}"
+    # Sourced in the runner's shell so break/fix/assert see the helpers;
+    # unset afterwards so the next scenario can't inherit stale functions.
+    # shellcheck disable=SC1090
+    source "$f"
+    local ok=true
+    step "break"
+    if ! scenario_break; then
+        error "$name: break step failed"
+        ok=false
+    fi
+    if [ "$ok" = "true" ]; then
+        step "fix"
+        if ! scenario_fix; then
+            error "$name: fix step failed"
+            ok=false
+        fi
+    fi
+    if [ "$ok" = "true" ]; then
+        step "assert"
+        if ! scenario_assert; then
+            error "$name: post-state assertion failed"
+            ok=false
+        fi
+    fi
+    unset -f scenario_break scenario_fix scenario_assert
+    unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES
+    if [ "$ok" = "true" ]; then
+        success "PASS: $name"
+        PASS+=("$name")
+    else
+        error "FAIL: $name"
+        FAIL+=("$name")
+    fi
+}
+
+section "Preparing VM"
+stop_qemu 2>/dev/null || true
+step "Restoring clean-install snapshot"
+restore_snapshot "$DISK_PATH" "clean-install" || fatal "restore failed"
+boot_vm
+bootstrap_vm
+
+step "Freezing bootstrap as snapshot: $READY_SNAPSHOT"
+stop_qemu
+snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT" \
+    && delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT"
+create_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "snapshot failed"
+
+first_group=true
+for g in "${GROUP_ORDER[@]}"; do
+    section "Group: $g"
+    if [ "$first_group" = "true" ]; then
+        first_group=false
+    else
+        step "Isolation boundary: restore $READY_SNAPSHOT"
+        stop_qemu
+        restore_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "restore failed"
+    fi
+    boot_vm
+    for i in "${!S_FILES[@]}"; do
+        [ "${S_GROUPS[$i]}" = "$g" ] && run_scenario "$i"
+    done
+    stop_qemu
+done
+
+# ─── Report ──────────────────────────────────────────────────────────
+
+section "Results"
+for n in "${PASS[@]}"; do success "PASS $n"; done
+for n in "${FAIL[@]}"; do error "FAIL $n"; done
+info "Log: $LOGFILE"
+
+if [ "$KEEP_VM" = "true" ]; then
+    warn "--keep: VM left in post-run state; base image NOT restored"
+    warn "restore by hand: qemu-img snapshot -a clean-install $DISK_PATH"
+    CLEANUP_DONE=1
+else
+    cleanup_scenarios
+fi
+
+if [ ${#FAIL[@]} -gt 0 ]; then
+    error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed"
+    exit 1
+fi
+success "All ${#PASS[@]} scenarios passed"
+exit 0