test(maint): add VM and nspawn remedy scenario harness

Nine break/fix/assert scenarios run the real maint fix inside the test VM. The runner batches non-conflicting scenarios into one VM boot, restores the snapshot only between groups, and leaves the base image pristine afterwards. A systemd-nspawn fast lane runs the pacman-level group against a cached pacstrap rootfs in seconds. The plan layer validates the scenario contract without a VM and carries a 19-test unit suite. make test-maint wires the VM lane in.

The zfs lane is filtered but unexercised: the FS_PROFILE=zfs base image fails to build (ZFS DKMS module not found on linux-lts 6.18). The failure is tracked in the todo.

1 files changed, 268 insertions, 0 deletions

diff --git a/scripts/testing/run-maint-nspawn.sh b/scripts/testing/run-maint-nspawn.sh
new file mode 100644
index 0000000..04d4a32
--- /dev/null
+++ b/scripts/testing/run-maint-nspawn.sh
@@ -0,0 +1,268 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Run pacman-level maint remedy scenarios in a systemd-nspawn container
+# Author: Craig Jennings <craigmartinjennings@gmail.com>
+# License: GNU GPLv3
+#
+# The fast lane for the maint scenario suite: pure pacman-level cases (the
+# "packages" scenario group — cache trims, orphan removal, pacnew deletion)
+# don't need a booted machine, so they run in seconds against a pacstrap'd
+# rootfs instead of minutes against the VM. Everything else (systemd, logs)
+# needs pid 1 and journald — that stays in run-maint-scenarios.sh.
+#
+# The scenario files are shared with the VM runner and see the same helpers
+# (mexec/mmaint/mfix/massert_metric); only the transport differs — the
+# helper trio is deliberately duplicated between the two runners rather
+# than abstracted over ssh-vs-nspawn.
+#
+# The rootfs is cached at $NSPAWN_ROOT and rebuilt with --fresh. Building
+# and running need root (pacstrap, systemd-nspawn) via sudo.
+#
+# Usage: run-maint-nspawn.sh [--list] [--fresh]
+#   --list   print the validated scenario plan and exit (no root needed)
+#   --fresh  delete the cached rootfs and pacstrap a new one
+# Env: MAINT_SCENARIO_DIR   scenario dir (default: maint-scenarios/)
+#      MAINT_SRC            maint package source tree
+#                           (default: ~/.dotfiles/maint/src/maint)
+#      NSPAWN_ROOT          rootfs dir (default: /var/tmp/archsetup-maint-nspawn)
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+source "$SCRIPT_DIR/lib/logging.sh"
+
+SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}"
+MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}"
+THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml"
+NSPAWN_ROOT="${NSPAWN_ROOT:-/var/tmp/archsetup-maint-nspawn}"
+NSPAWN_GROUP="packages"
+
+LIST_ONLY=false
+FRESH=false
+
+usage() {
+    echo "Usage: $0 [--list] [--fresh]"
+    echo "  --list   print the validated scenario plan and exit"
+    echo "  --fresh  rebuild the cached rootfs from scratch"
+    echo "Env: MAINT_SCENARIO_DIR, MAINT_SRC, NSPAWN_ROOT"
+}
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --list)  LIST_ONLY=true; shift ;;
+        --fresh) FRESH=true; shift ;;
+        *) usage; exit 1 ;;
+    esac
+done
+
+# ─── Plan: the pacman-level subset ───────────────────────────────────
+# Same contract validation as the VM runner; the selection rule is the
+# group name — "packages" is pacman-level by construction.
+
+S_FILES=() S_NAMES=() S_DESCS=()
+
+_scenario_var() {  # <file> <varname>
+    bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \
+        _probe "$1" "$2" 2>/dev/null
+}
+
+_validate_scenario() {  # <file> -> fatal on contract violation
+    local f="$1" base probe_err
+    base=$(basename "$f")
+    probe_err=$(bash -c 'set -eu; source "$1"
+        : "${SCENARIO_DESC:?missing SCENARIO_DESC}"
+        : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}"
+        : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}"
+        for p in $SCENARIO_PROFILES; do
+            case "$p" in btrfs|zfs|any) ;; *)
+                echo "bad profile token: $p" >&2; exit 1 ;;
+            esac
+        done
+        declare -f scenario_break scenario_fix scenario_assert >/dev/null \
+            || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \
+        _probe "$f" 2>&1 >/dev/null) \
+        || fatal "scenario contract violation in $base: $probe_err"
+}
+
+build_plan() {
+    [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR"
+    local f base
+    for f in "$SCENARIO_DIR"/*.sh; do
+        [ -e "$f" ] || break
+        _validate_scenario "$f"
+        [ "$(_scenario_var "$f" SCENARIO_GROUP)" = "$NSPAWN_GROUP" ] || continue
+        base=$(basename "$f" .sh)
+        S_FILES+=("$f")
+        S_NAMES+=("${base#[0-9][0-9]-}")
+        S_DESCS+=("$(_scenario_var "$f" SCENARIO_DESC)")
+    done
+    [ ${#S_FILES[@]} -gt 0 ] \
+        || fatal "no '$NSPAWN_GROUP'-group scenarios in $SCENARIO_DIR"
+}
+
+print_plan() {
+    local i
+    echo "maint nspawn scenario plan (group: $NSPAWN_GROUP)"
+    for i in "${!S_FILES[@]}"; do
+        echo "  ${S_NAMES[$i]} — ${S_DESCS[$i]}"
+    done
+}
+
+if [ "$LIST_ONLY" = "true" ]; then
+    LOGFILE=/dev/null
+    build_plan
+    print_plan
+    exit 0
+fi
+
+# ─── Run ─────────────────────────────────────────────────────────────
+
+TIMESTAMP=$(date +'%Y%m%d-%H%M%S')
+RESULTS_DIR="$PROJECT_ROOT/test-results/maint-nspawn-$TIMESTAMP"
+mkdir -p "$RESULTS_DIR"
+LOGFILE="$RESULTS_DIR/scenarios.log"
+init_logging "$LOGFILE"
+
+build_plan
+
+section "Maint Remedy Scenarios (nspawn): $TIMESTAMP"
+info "Rootfs: $NSPAWN_ROOT"
+info "Maint source: $MAINT_SRC"
+print_plan | tee -a "$LOGFILE"
+
+command -v pacstrap >/dev/null \
+    || fatal "pacstrap not found — pacman -S arch-install-scripts"
+command -v systemd-nspawn >/dev/null || fatal "systemd-nspawn not found"
+[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC"
+[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML"
+sudo -n true 2>/dev/null || fatal "needs passwordless sudo (pacstrap/nspawn)"
+
+# ─── Target helpers (available to scenario scripts) ──────────────────
+
+mexec() {  # run a command in the container as root
+    sudo systemd-nspawn -q -D "$NSPAWN_ROOT" --pipe \
+        /bin/bash -c "$*" 2>> "$LOGFILE"
+}
+
+mmaint() {
+    mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*"
+}
+
+mfix() {
+    local id="$1"; shift || true
+    step "maint fix $id ${*:+$* }--dry-run"
+    mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \
+        || { error "dry-run failed: maint fix $id $*"; return 1; }
+    step "maint fix $id $*"
+    mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE"
+    return "${PIPESTATUS[0]}"
+}
+
+massert_metric() {  # <metric-id> <expected-severity>
+    local mid="$1" want="$2" got
+    got=$(mmaint status --json | python3 -c "
+import json, sys
+env = json.load(sys.stdin)
+ms = [m for m in env['metrics'] if m['id'] == '$mid']
+print(ms[0]['severity'] if ms else 'MISSING')
+")
+    if [ "$got" = "$want" ]; then
+        success "metric $mid is $want"
+        return 0
+    fi
+    error "metric $mid: expected $want, got $got"
+    return 1
+}
+
+# ─── Rootfs ──────────────────────────────────────────────────────────
+
+if [ "$FRESH" = "true" ] && [ -d "$NSPAWN_ROOT" ]; then
+    step "Removing cached rootfs (--fresh)"
+    sudo rm -rf "$NSPAWN_ROOT"
+fi
+
+if [ ! -x "$NSPAWN_ROOT/usr/bin/python3" ]; then
+    section "Building rootfs (pacstrap)"
+    sudo mkdir -p "$NSPAWN_ROOT"
+    # shellcheck disable=SC2024  # the log is user-owned by design
+    sudo pacstrap -c -K "$NSPAWN_ROOT" base python pacman-contrib expac \
+        >> "$LOGFILE" 2>&1 || fatal "pacstrap failed (see $LOGFILE)"
+    success "Rootfs built"
+else
+    info "Reusing cached rootfs (--fresh to rebuild)"
+fi
+
+section "Installing maint tree + thresholds into the container"
+sudo rm -rf "$NSPAWN_ROOT/root/maint-pkg"
+sudo mkdir -p "$NSPAWN_ROOT/root/maint-pkg" "$NSPAWN_ROOT/root/.config/archsetup"
+sudo cp -r "$MAINT_SRC" "$NSPAWN_ROOT/root/maint-pkg/"
+sudo find "$NSPAWN_ROOT/root/maint-pkg" -name '__pycache__' -type d \
+    -exec rm -rf {} + 2>/dev/null || true
+sudo cp "$THRESHOLDS_TOML" \
+    "$NSPAWN_ROOT/root/.config/archsetup/maintenance-thresholds.toml"
+
+step "Smoke: maint status --json runs in the container"
+mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \
+    || fatal "maint status failed in the container"
+success "Container ready"
+
+# ─── Execution ───────────────────────────────────────────────────────
+
+PASS=() FAIL=()
+
+run_scenario() {  # <index>
+    local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}"
+    section "Scenario: $name — ${S_DESCS[$1]}"
+    # shellcheck disable=SC1090
+    source "$f"
+    local ok=true
+    step "break"
+    if ! scenario_break; then
+        error "$name: break step failed"
+        ok=false
+    fi
+    if [ "$ok" = "true" ]; then
+        step "fix"
+        if ! scenario_fix; then
+            error "$name: fix step failed"
+            ok=false
+        fi
+    fi
+    if [ "$ok" = "true" ]; then
+        step "assert"
+        if ! scenario_assert; then
+            error "$name: post-state assertion failed"
+            ok=false
+        fi
+    fi
+    unset -f scenario_break scenario_fix scenario_assert
+    unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES
+    if [ "$ok" = "true" ]; then
+        success "PASS: $name"
+        PASS+=("$name")
+    else
+        error "FAIL: $name"
+        FAIL+=("$name")
+    fi
+}
+
+for i in "${!S_FILES[@]}"; do
+    run_scenario "$i"
+done
+
+# ─── Report ──────────────────────────────────────────────────────────
+
+section "Results"
+for n in "${PASS[@]}"; do success "PASS $n"; done
+for n in "${FAIL[@]}"; do error "FAIL $n"; done
+info "Log: $LOGFILE"
+info "Rootfs kept for reuse: $NSPAWN_ROOT (--fresh to rebuild)"
+
+if [ ${#FAIL[@]} -gt 0 ]; then
+    error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed"
+    exit 1
+fi
+success "All ${#PASS[@]} scenarios passed"
+exit 0