test(maint): add VM and nspawn remedy scenario harness

Nine break/fix/assert scenarios run the real maint fix inside the test VM. The runner batches non-conflicting scenarios into one VM boot, restores the snapshot only between groups, and leaves the base image pristine afterwards. A systemd-nspawn fast lane runs the pacman-level group against a cached pacstrap rootfs in seconds. The plan layer validates the scenario contract without a VM and carries a 19-test unit suite. make test-maint wires the VM lane in.

The zfs lane is filtered but unexercised: the FS_PROFILE=zfs base image fails to build (ZFS DKMS module not found on linux-lts 6.18). The failure is tracked in the todo.

9 files changed, 209 insertions, 0 deletions

diff --git a/scripts/testing/maint-scenarios/10-journal-vacuum.sh b/scripts/testing/maint-scenarios/10-journal-vacuum.sh
new file mode 100644
index 0000000..b61de49
--- /dev/null
+++ b/scripts/testing/maint-scenarios/10-journal-vacuum.sh
@@ -0,0 +1,30 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: the journal vacuum executes against live journald at the TOML floor.
+#
+# What this proves: the TOML edit reaches the remedy's argv (vacuum target
+# and warn threshold are the same key), journalctl --vacuum-size runs for
+# real against live journald, and the metric grades ok against the edited
+# floor afterwards.
+#
+# What it deliberately does NOT prove: size reduction. The key is int-GB and
+# bottoms out at 1, vacuum only touches archived files, and a VM-sized
+# journal never nears 1 GB — a genuine reduction test would need journald
+# rate-limit surgery plus ~1 GB of spam per run. A pre/post usage comparison
+# is also unsound here: journald keeps landing in-flight writes (the spam is
+# async, and every ssh hop logs), so usage can grow across a correct vacuum.
+SCENARIO_DESC="journal vacuum runs at the TOML size floor"
+SCENARIO_GROUP="logs"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "sed -i 's/^journal_disk_warn_gb *=.*/journal_disk_warn_gb = 1/' /root/.config/archsetup/maintenance-thresholds.toml" \
+        && mexec "for i in \$(seq 1 20); do head -c 900000 /dev/urandom | base64 | systemd-cat -t maint-scenario-spam; done; journalctl --rotate; journalctl --flush >/dev/null 2>&1 || true"
+}
+
+scenario_fix() {
+    mfix journal_vacuum
+}
+
+scenario_assert() {
+    massert_metric journal_disk ok
+}
diff --git a/scripts/testing/maint-scenarios/11-coredump-age-out.sh b/scripts/testing/maint-scenarios/11-coredump-age-out.sh
new file mode 100644
index 0000000..06d6fa0
--- /dev/null
+++ b/scripts/testing/maint-scenarios/11-coredump-age-out.sh
@@ -0,0 +1,21 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: coredump files beyond the forensic window are aged out.
+#
+# coredumpctl has no clean verb, so the remedy is a find -mtime +N -delete
+# over /var/lib/systemd/coredump. Break plants one file older than the
+# window and one fresh file; assert checks exactly the old one is gone.
+SCENARIO_DESC="aged coredump files removed, fresh ones kept"
+SCENARIO_GROUP="logs"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "mkdir -p /var/lib/systemd/coredump && touch -d '30 days ago' '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && touch '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'"
+}
+
+scenario_fix() {
+    mfix coredump_clean
+}
+
+scenario_assert() {
+    mexec "test ! -e '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && test -e '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'"
+}
diff --git a/scripts/testing/maint-scenarios/20-cache-keep3.sh b/scripts/testing/maint-scenarios/20-cache-keep3.sh
new file mode 100644
index 0000000..a9a6ee5
--- /dev/null
+++ b/scripts/testing/maint-scenarios/20-cache-keep3.sh
@@ -0,0 +1,24 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: paccache -rk3 trims an installed package's cache to 3 versions.
+#
+# paccache decides from filenames, so five fake versions of a package that
+# IS installed (bash) stand in for a fat cache. The bootstrap may leave a
+# real bash package in the cache too, so the assert uses the ordering
+# invariant rather than a count: the two newest fakes survive any keep-3
+# window that contains them, and the two oldest fall outside it whether or
+# not a real (newer) version is cached alongside.
+SCENARIO_DESC="pacman cache trimmed to three versions per package"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "for v in 1.0.1 1.0.2 1.0.3 1.0.4 1.0.5; do touch /var/cache/pacman/pkg/bash-\$v-1-x86_64.pkg.tar.zst; done; ls /var/cache/pacman/pkg/ | grep -c '^bash-'"
+}
+
+scenario_fix() {
+    mfix cache_clean
+}
+
+scenario_assert() {
+    mexec "test -e /var/cache/pacman/pkg/bash-1.0.5-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-1.0.4-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.2-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.1-1-x86_64.pkg.tar.zst"
+}
diff --git a/scripts/testing/maint-scenarios/21-cache-uninstalled.sh b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh
new file mode 100644
index 0000000..ea788e8
--- /dev/null
+++ b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh
@@ -0,0 +1,20 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: paccache -ruk0 drops every cached version of uninstalled packages.
+#
+# Fake versions of a package that is NOT installed disappear entirely, while
+# an installed package's cached version survives the uninstalled-only pass.
+SCENARIO_DESC="cached versions of uninstalled packages dropped"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "touch /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst"
+}
+
+scenario_fix() {
+    mfix cache_clean_uninstalled
+}
+
+scenario_assert() {
+    mexec "test ! -e /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst"
+}
diff --git a/scripts/testing/maint-scenarios/22-orphan-remove.sh b/scripts/testing/maint-scenarios/22-orphan-remove.sh
new file mode 100644
index 0000000..8412128
--- /dev/null
+++ b/scripts/testing/maint-scenarios/22-orphan-remove.sh
@@ -0,0 +1,22 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: a real orphan is created, detected, and removed by name.
+#
+# Installing `tree` --asdeps with no dependents makes it a true orphan
+# (pacman -Qtdq lists it). The remedy is item-based: maint fix orphan_remove
+# tree. Assert covers both the package state and the metric verdict.
+SCENARIO_DESC="orphan package removed via the item-based remedy"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "pacman -S --noconfirm --asdeps tree && pacman -Qtdq | grep -qx tree"
+}
+
+scenario_fix() {
+    mfix orphan_remove tree
+}
+
+scenario_assert() {
+    mexec "! pacman -Qi tree >/dev/null 2>&1 && ! pacman -Qtdq | grep -qx tree" \
+        && massert_metric pkg_orphans ok
+}
diff --git a/scripts/testing/maint-scenarios/23-pacnew-delete.sh b/scripts/testing/maint-scenarios/23-pacnew-delete.sh
new file mode 100644
index 0000000..23cd8d1
--- /dev/null
+++ b/scripts/testing/maint-scenarios/23-pacnew-delete.sh
@@ -0,0 +1,22 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: a .pacnew file is deleted through the validated priv verb.
+#
+# pacdiff --output finds the planted file; the remedy takes the path as its
+# item and rm's it. A neighbouring non-pacnew file proves the verb's path
+# validation keeps the blast radius to the named file.
+SCENARIO_DESC="planted .pacnew removed by path"
+SCENARIO_GROUP="packages"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "cp /etc/pacman.conf /etc/pacman.conf.pacnew && pacdiff --output | grep -q pacman.conf.pacnew"
+}
+
+scenario_fix() {
+    mfix pacnew_delete /etc/pacman.conf.pacnew
+}
+
+scenario_assert() {
+    mexec "test ! -e /etc/pacman.conf.pacnew && test -e /etc/pacman.conf" \
+        && massert_metric pkg_pacnew ok
+}
diff --git a/scripts/testing/maint-scenarios/30-unit-reset.sh b/scripts/testing/maint-scenarios/30-unit-reset.sh
new file mode 100644
index 0000000..07eed53
--- /dev/null
+++ b/scripts/testing/maint-scenarios/30-unit-reset.sh
@@ -0,0 +1,28 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: a failed unit's state is reset.
+#
+# A transient oneshot that exits 1 lands in systemctl --failed; the remedy
+# resets it and the failed list returns to its pre-break size. The unit name
+# rides the remedy's item argument, exercising the priv verb's unit-name
+# validation. The base image carries its own failed unit (grub-btrfsd — no
+# snapshot dirs in the VM), so the assert scopes to this scenario's unit and
+# the count delta, never the absolute failed list.
+SCENARIO_DESC="failed unit reset clears the failed list"
+SCENARIO_GROUP="systemd"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    PRE_FAILED_COUNT=$(mexec "systemctl --failed --no-legend | wc -l") \
+        && mexec "systemd-run --unit=maint-scenario-fail --service-type=oneshot --no-block /bin/false; sleep 2; systemctl is-failed maint-scenario-fail.service"
+}
+
+scenario_fix() {
+    mfix unit_reset maint-scenario-fail.service
+}
+
+scenario_assert() {
+    local pre="$PRE_FAILED_COUNT"
+    unset PRE_FAILED_COUNT  # don't leak scenario state past this scenario
+    mexec "! systemctl is-failed maint-scenario-fail.service >/dev/null 2>&1" \
+        && [ "$(mexec "systemctl --failed --no-legend | wc -l")" = "$pre" ]
+}
diff --git a/scripts/testing/maint-scenarios/31-fstrim-enable.sh b/scripts/testing/maint-scenarios/31-fstrim-enable.sh
new file mode 100644
index 0000000..2d548a1
--- /dev/null
+++ b/scripts/testing/maint-scenarios/31-fstrim-enable.sh
@@ -0,0 +1,22 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: fstrim.timer enabled and started from the disabled default.
+#
+# The base image ships util-linux with fstrim.timer disabled — the exact
+# state the metric flags. The remedy is systemctl enable --now.
+SCENARIO_DESC="fstrim.timer enabled and started"
+SCENARIO_GROUP="systemd"
+SCENARIO_PROFILES="btrfs"
+
+scenario_break() {
+    # Default state, but pin it so the scenario stays honest if a future
+    # base image enables the timer.
+    mexec "systemctl disable --now fstrim.timer >/dev/null 2>&1 || true; ! systemctl is-enabled fstrim.timer >/dev/null 2>&1"
+}
+
+scenario_fix() {
+    mfix fstrim_enable
+}
+
+scenario_assert() {
+    mexec "systemctl is-enabled fstrim.timer && systemctl is-active fstrim.timer"
+}
diff --git a/scripts/testing/maint-scenarios/32-cron-enable.sh b/scripts/testing/maint-scenarios/32-cron-enable.sh
new file mode 100644
index 0000000..39d4de6
--- /dev/null
+++ b/scripts/testing/maint-scenarios/32-cron-enable.sh
@@ -0,0 +1,20 @@
+# shellcheck shell=bash disable=SC2034  # sourced by run-maint-scenarios.sh
+# Scenario: cronie enabled and started (the stopped-cron posture).
+#
+# Bootstrap installs cronie but never enables it — the broken state is the
+# package's default. The remedy enables + starts the service.
+SCENARIO_DESC="cronie enabled and started from the stopped state"
+SCENARIO_GROUP="systemd"
+SCENARIO_PROFILES="any"
+
+scenario_break() {
+    mexec "systemctl disable --now cronie.service >/dev/null 2>&1 || true; ! systemctl is-active cronie.service >/dev/null 2>&1"
+}
+
+scenario_fix() {
+    mfix cron_enable
+}
+
+scenario_assert() {
+    mexec "systemctl is-enabled cronie.service && systemctl is-active cronie.service"
+}