diff options
Diffstat (limited to 'scripts')
19 files changed, 1287 insertions, 24 deletions
diff --git a/scripts/hypr-live-update-guard b/scripts/hypr-live-update-guard new file mode 100755 index 0000000..e78200d --- /dev/null +++ b/scripts/hypr-live-update-guard @@ -0,0 +1,139 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later +# hypr-live-update-guard - abort a live GPU/compositor library upgrade. +# +# Installed as a pacman PreTransaction hook. When an upgrade transaction +# would CHANGE the on-disk version of GPU/compositor runtime libraries +# (mesa, hyprland, wayland, GPU drivers, ...) AND a Hyprland session is +# running, this aborts the transaction BEFORE any package is swapped. +# Replacing those libraries out from under a live compositor makes the next +# GPU-lib call hit a now "(deleted)" file and SIGABRT, taking the Wayland +# clients down with it (hit on ratio 2026-06-07: mesa + hyprland upgraded +# live, Hyprland crashed and took awww/insync/emacs with it). Aborting at +# PreTransaction is the safe point: nothing has been replaced yet, so the +# running session is untouched and the user can re-run from a TTY. +# +# Version-aware (2026-07-08): a same-version reinstall writes identical +# bytes over identical bytes and is harmless to the live session — the +# maintenance console's integrity REINSTALL is exactly that after an +# update, and the name-only guard was blocking it. Each target's installed +# version (pacman -Q) is compared against the sync-db candidate +# (expac -S %v); targets that match are pure reinstalls and pass. Both are +# read-only queries and run fine inside a hook — the ALPM lock only gates +# other transactions (pinned live with db.lck present). A target whose +# versions can't be resolved (AUR package, a -U transaction installing a +# local file, expac absent) reads as unknown and blocks conservatively — +# a -U of a guard-listed package at repo-matching version would slip +# through, which is accepted as pathological. +# +# Pacman feeds the matched package names on stdin (NeedsTargets). +# +# Test seams / overrides (env): +# HYPR_GUARD_RUNNING 1/0 forces the running check (default: pgrep Hyprland) +# HYPR_ALLOW_LIVE_UPDATE 1 proceeds anyway (skip the guard) +# HYPR_GUARD_SENTINEL path whose existence also proceeds anyway +# (default /run/archsetup-allow-live-gpu-update, +# cleared on reboot since /run is tmpfs) +# HYPR_GUARD_VERSIONS "pkg installed candidate" lines replacing the +# pacman/expac lookups; when set, a package absent +# from the map reads as unknown (blocks) + +set -u + +sentinel="${HYPR_GUARD_SENTINEL:-/run/archsetup-allow-live-gpu-update}" + +# Explicit override: the user knows what they're doing. The sentinel is +# consumed as it's honored — one touch buys exactly one transaction, so a +# leftover from a crashed caller can't keep the guard disarmed until +# reboot. +if [ "${HYPR_ALLOW_LIVE_UPDATE:-0}" = "1" ]; then + exit 0 +fi +if [ -e "$sentinel" ]; then + rm -f "$sentinel" 2>/dev/null || true + exit 0 +fi + +hyprland_running() { + if [ -n "${HYPR_GUARD_RUNNING:-}" ]; then + [ "$HYPR_GUARD_RUNNING" = "1" ] + return + fi + pgrep -x Hyprland >/dev/null 2>&1 +} + +# No live session means no live swap to worry about. Let the upgrade run -- +# this is exactly the from-a-TTY-after-logout path the warning points to. +hyprland_running || exit 0 + +# Collect the triggering packages (stdin from NeedsTargets). +targets=$(cat 2>/dev/null | sort -u) + +# "installed candidate" for a package, or nothing when unknown. +versions_for() { + if [ -n "${HYPR_GUARD_VERSIONS+x}" ]; then + printf '%s\n' "$HYPR_GUARD_VERSIONS" \ + | awk -v p="$1" '$1 == p { print $2, $3; exit }' + else + inst=$(pacman -Q -- "$1" 2>/dev/null | awk '{ print $2 }') + # expac prints one line per repo carrying the package; the first is + # the repo pacman resolves from + cand=$(expac -S '%v' -- "$1" 2>/dev/null | head -n 1) + printf '%s %s\n' "$inst" "$cand" + fi +} + +# Split the targets into version-changing (dangerous live) and same-version +# reinstalls (harmless). Unknown versions count as dangerous. +changed="" +count=0 +for pkg in $targets; do + count=$((count + 1)) + # shellcheck disable=SC2046 # splitting "inst cand" into $1 $2 is the point + set -- $(versions_for "$pkg") + inst=${1:-} + cand=${2:-} + if [ -n "$inst" ] && [ -n "$cand" ] && [ "$inst" = "$cand" ]; then + continue # pure reinstall: identical bytes, no swap hazard + fi + changed="$changed $pkg" +done + +# Every guarded target is a pure reinstall -- nothing changes on disk that +# the live session has mapped. Let it through silently. +if [ "$count" -gt 0 ] && [ -z "$changed" ]; then + exit 0 +fi + +# An empty target list shouldn't normally happen (the hook only fires when +# dangerous targets exist); if Hyprland is up, stay safe and abort. +pkg_lines="" +for pkg in $changed; do + pkg_lines="${pkg_lines} - ${pkg} +" +done +[ -n "$pkg_lines" ] || pkg_lines=" (GPU/compositor runtime libraries) +" + +cat >&2 <<EOF + +========================================================================== + BLOCKED: live GPU/compositor library upgrade while Hyprland is running +========================================================================== + Packages in this upgrade change version and can crash the running + compositor if swapped now: +${pkg_lines} + Replacing these out from under a live Hyprland session makes the next + GPU-lib call hit a deleted library and SIGABRT, taking your Wayland apps + down with it (and risking an unclean shutdown). + + Do it safely instead -- from a TTY with Hyprland stopped: + 1. Log out of Hyprland, or switch to a console (Ctrl+Alt+F2) and log in. + 2. Re-run the upgrade there: sudo pacman -Syu + + To override and proceed anyway (not recommended while Hyprland runs): + sudo touch $sentinel && sudo pacman -Syu +========================================================================== + +EOF +exit 1 diff --git a/scripts/import-wireguard-configs.sh b/scripts/import-wireguard-configs.sh new file mode 100755 index 0000000..9e42033 --- /dev/null +++ b/scripts/import-wireguard-configs.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Import the assets/wireguard-config Proton configs into NetworkManager as +# wireguard connections with autoconnect off. Two NM quirks handled here: +# +# - The import filename must be a valid interface name (<= 15 chars), and +# several config names are longer — so every file imports through a temp +# copy named wgpvpn.conf and the connection is renamed to the real config +# name right after (by the UUID parsed from the import output, so a stray +# same-named connection can't be hit). All profiles share the wgpvpn +# interface, which is fine (they're mutually exclusive full-tunnel +# configs), and the wg prefix keeps the net doctor's tunnel-down repair +# on the NM path. +# - `nmcli connection import` both sets autoconnect yes AND activates the +# profile immediately. These are full-tunnel (AllowedIPs 0.0.0.0/0), so a +# bare import silently brings the tunnel up and routes everything through it +# — a VPN nobody asked for by importing a file. Each import is therefore +# followed by a modify (autoconnect no) and a deactivate (connection down), +# so importing only ever adds inactive profiles. +# +# A connection still literally named wgpvpn means an earlier run died +# between import and rename — and it still has autoconnect on. The script +# refuses to run until that's cleaned up rather than guessing. +# +# Idempotent: already-imported names skip. +# +# Usage: import-wireguard-configs.sh [config-dir] +set -euo pipefail + +dir="${1:-$(cd "$(dirname "$0")/.." && pwd)/assets/wireguard-config}" +[ -d "$dir" ] || { echo "no such config dir: $dir" >&2; exit 1; } + +if nmcli -t -f NAME connection show | grep -Fxq "wgpvpn"; then + echo "stale 'wgpvpn' connection found (an earlier run died mid-import; it has autoconnect ON)" >&2 + echo "inspect and remove it first: nmcli connection delete wgpvpn" >&2 + exit 1 +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +shopt -s nullglob +found=0 +for conf in "$dir"/*.conf; do + found=1 + name="$(basename "$conf" .conf)" + if nmcli -t -f NAME connection show | grep -Fxq "$name"; then + echo "skip: $name (already imported)" + continue + fi + cp "$conf" "$tmp/wgpvpn.conf" + out="$(nmcli connection import type wireguard file "$tmp/wgpvpn.conf")" + uuid="$(grep -oE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' <<<"$out" | head -1 || true)" + if [ -z "$uuid" ]; then + echo "could not parse a UUID from the import output for $name:" >&2 + echo " $out" >&2 + exit 1 + fi + nmcli connection modify "$uuid" connection.id "$name" \ + connection.autoconnect no + # nmcli import auto-activates; bring it back down so importing never leaves + # a tunnel up. A profile that didn't activate makes this a harmless no-op. + nmcli connection down "$uuid" >/dev/null 2>&1 || true + echo "imported: $name (inactive, autoconnect off, iface wgpvpn)" +done +[ "$found" = 1 ] || { echo "no .conf files in $dir" >&2; exit 1; } diff --git a/scripts/testing/lib/network-diagnostics.sh b/scripts/testing/lib/network-diagnostics.sh index 38788e5..dc54334 100644 --- a/scripts/testing/lib/network-diagnostics.sh +++ b/scripts/testing/lib/network-diagnostics.sh @@ -6,58 +6,110 @@ # Note: logging.sh and vm-utils.sh should already be sourced by the calling script # Uses globals: ROOT_PASSWORD, SSH_PORT, SSH_OPTS, VM_IP (from vm-utils.sh or calling script) +# Optional global: TEST_RESULTS_DIR (raw command outputs are saved there when set) -# Run quick network diagnostics +# Gather one read-only fact from the VM, print it, and save the raw output. +# Facts are collected regardless of pass/fail so a failing install still leaves +# the IP/route/resolver evidence in the log and the results dir. +# $1 label human-readable label for the fact +# $2 slug filename slug for the saved raw output +# $3 cmd remote command to run over the shared ssh_base +# Uses the caller's locals ssh_base and results_dir (dynamic scope). +_netdiag_fact() { + local label="$1" slug="$2" cmd="$3" out + out="$($ssh_base "$cmd" 2>&1)" + info "${label}:" + printf '%s\n' "$out" | while IFS= read -r line; do + info " $line" + done + if [ -n "$results_dir" ]; then + printf '%s\n' "$out" > "$results_dir/netdiag-${slug}.txt" 2>/dev/null || true + fi +} + +# Run quick network diagnostics. +# +# Evidence first: collect read-only facts (interfaces, route, resolver) +# unconditionally, then run every reachability check and report all failures at +# the end. A DNS failure is named as a DNS failure, not masked as a generic "no +# internet" or misattributed to the Arch mirror. Returns 0 when all checks pass, +# non-zero when any check fails, so callers keep their success/failure contract. run_network_diagnostics() { local password="${ROOT_PASSWORD:-archsetup}" local port="${SSH_PORT:-22}" local host="${VM_IP:-localhost}" local ssh_base="sshpass -p $password ssh $SSH_OPTS -p $port root@$host" + local results_dir="${TEST_RESULTS_DIR:-}" + local failures=() section "Pre-flight Network Diagnostics" - # Test 1: Basic connectivity (use curl instead of ping - SLIRP may not handle ICMP) - step "Testing internet connectivity" - if $ssh_base "curl -s --connect-timeout 5 -o /dev/null http://archlinux.org" 2>/dev/null; then - success "Internet connectivity OK" - else - error "No internet connectivity" - return 1 - fi + # --- Phase 1: collect read-only facts, unconditionally --- + # These never gate the outcome; they exist so a failed install still has + # the interface/route/resolver evidence to diagnose from. + step "Collecting interface addresses" + _netdiag_fact "Interface addresses (ip -brief addr)" "ip-addr" "ip -brief addr" + + step "Collecting default route" + _netdiag_fact "Default route (ip route show default)" "ip-route" "ip route show default" - # Test 2: DNS resolution (use getent which is always available, unlike nslookup/dig) + step "Reading resolver configuration" + _netdiag_fact "Resolver (/etc/resolv.conf)" "resolv-conf" "cat /etc/resolv.conf" + + # --- Phase 2: generic connectivity checks (run all, don't short-circuit) --- + # DNS, egress, and TLS are independent failure modes. Keeping them separate + # means a resolver problem reads as DNS, not as a downstream mirror failure. step "Testing DNS resolution" if $ssh_base "getent hosts archlinux.org >/dev/null 2>&1" 2>/dev/null; then success "DNS resolution OK" else error "DNS resolution failed" - return 1 + failures+=("DNS resolution (getent hosts archlinux.org)") fi - # Test 3: Arch mirror accessibility + step "Testing HTTP egress" + if $ssh_base "curl -s --connect-timeout 5 -o /dev/null http://archlinux.org" 2>/dev/null; then + success "HTTP egress OK" + else + error "HTTP egress failed" + failures+=("HTTP egress (http://archlinux.org)") + fi + + step "Testing TLS/HTTPS egress" + if $ssh_base "curl -s --connect-timeout 5 -o /dev/null https://archlinux.org" 2>/dev/null; then + success "TLS/HTTPS egress OK" + else + error "TLS/HTTPS egress failed" + failures+=("TLS/HTTPS egress (https://archlinux.org)") + fi + + # --- Phase 3: Arch-specific checks (run all, don't short-circuit) --- step "Testing Arch mirror access" if $ssh_base "curl -s -I https://geo.mirror.pkgbuild.com/ | head -1 | grep -qE '(200|301|302)'" 2>/dev/null; then success "Arch mirrors accessible" else error "Cannot reach Arch mirrors" - return 1 + failures+=("Arch mirror (https://geo.mirror.pkgbuild.com/)") fi - # Test 4: AUR accessibility step "Testing AUR access" if $ssh_base "curl -s -I https://aur.archlinux.org/ | head -1 | grep -qE '(200|405)'" 2>/dev/null; then success "AUR accessible" else error "Cannot reach AUR" - return 1 + failures+=("AUR (https://aur.archlinux.org/)") fi - # Show network info - info "Network configuration:" - $ssh_base "ip addr show | grep 'inet ' | grep -v '127.0.0.1'" 2>/dev/null | while IFS= read -r line; do - info " $line" - done + # --- Summary: report every failure, not just the first --- + if [ ${#failures[@]} -eq 0 ]; then + success "Network diagnostics complete - all checks passed" + return 0 + fi - success "Network diagnostics complete" - return 0 + error "Network diagnostics found ${#failures[@]} failure(s):" + local f + for f in "${failures[@]}"; do + error " - $f" + done + return 1 } diff --git a/scripts/testing/lib/vm-utils.sh b/scripts/testing/lib/vm-utils.sh index 10c0ca5..b85e773 100755 --- a/scripts/testing/lib/vm-utils.sh +++ b/scripts/testing/lib/vm-utils.sh @@ -11,7 +11,9 @@ # VM configuration defaults VM_CPUS="${VM_CPUS:-4}" -VM_RAM="${VM_RAM:-4096}" # MB +# 8 GiB headroom for AUR builds: makepkg runs -j$VM_CPUS, and parallel cc1plus +# (~700 MB each on heavy C++ packages) OOM-killed under the old 4 GiB default. +VM_RAM="${VM_RAM:-8192}" # MB VM_DISK_SIZE="${VM_DISK_SIZE:-50}" # GB # Filesystem profile: selects which base image + archangel config the harness @@ -59,7 +61,11 @@ init_vm_paths() { local img_suffix="" [ "$FS_PROFILE" != "btrfs" ] && img_suffix="-$FS_PROFILE" DISK_PATH="$VM_IMAGES_DIR/archsetup-base${img_suffix}.qcow2" - OVMF_VARS="$VM_IMAGES_DIR/OVMF_VARS.fd" + # Per-profile NVRAM: UEFI boot entries live here, outside the qcow2, so a + # disk-snapshot revert can't restore them. Sharing one file across profiles + # let a zfs run's ZFSBootMenu entries clobber the btrfs GRUB entry, leaving + # the btrfs base unbootable (no removable ESP fallback to recover from). + OVMF_VARS="$VM_IMAGES_DIR/OVMF_VARS${img_suffix}.fd" PID_FILE="$VM_IMAGES_DIR/qemu.pid" MONITOR_SOCK="$VM_IMAGES_DIR/qemu-monitor.sock" SERIAL_LOG="$VM_IMAGES_DIR/qemu-serial.log" diff --git a/scripts/testing/maint-scenarios/10-journal-vacuum.sh b/scripts/testing/maint-scenarios/10-journal-vacuum.sh new file mode 100644 index 0000000..b61de49 --- /dev/null +++ b/scripts/testing/maint-scenarios/10-journal-vacuum.sh @@ -0,0 +1,30 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: the journal vacuum executes against live journald at the TOML floor. +# +# What this proves: the TOML edit reaches the remedy's argv (vacuum target +# and warn threshold are the same key), journalctl --vacuum-size runs for +# real against live journald, and the metric grades ok against the edited +# floor afterwards. +# +# What it deliberately does NOT prove: size reduction. The key is int-GB and +# bottoms out at 1, vacuum only touches archived files, and a VM-sized +# journal never nears 1 GB — a genuine reduction test would need journald +# rate-limit surgery plus ~1 GB of spam per run. A pre/post usage comparison +# is also unsound here: journald keeps landing in-flight writes (the spam is +# async, and every ssh hop logs), so usage can grow across a correct vacuum. +SCENARIO_DESC="journal vacuum runs at the TOML size floor" +SCENARIO_GROUP="logs" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "sed -i 's/^journal_disk_warn_gb *=.*/journal_disk_warn_gb = 1/' /root/.config/archsetup/maintenance-thresholds.toml" \ + && mexec "for i in \$(seq 1 20); do head -c 900000 /dev/urandom | base64 | systemd-cat -t maint-scenario-spam; done; journalctl --rotate; journalctl --flush >/dev/null 2>&1 || true" +} + +scenario_fix() { + mfix journal_vacuum +} + +scenario_assert() { + massert_metric journal_disk ok +} diff --git a/scripts/testing/maint-scenarios/11-coredump-age-out.sh b/scripts/testing/maint-scenarios/11-coredump-age-out.sh new file mode 100644 index 0000000..06d6fa0 --- /dev/null +++ b/scripts/testing/maint-scenarios/11-coredump-age-out.sh @@ -0,0 +1,21 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: coredump files beyond the forensic window are aged out. +# +# coredumpctl has no clean verb, so the remedy is a find -mtime +N -delete +# over /var/lib/systemd/coredump. Break plants one file older than the +# window and one fresh file; assert checks exactly the old one is gone. +SCENARIO_DESC="aged coredump files removed, fresh ones kept" +SCENARIO_GROUP="logs" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "mkdir -p /var/lib/systemd/coredump && touch -d '30 days ago' '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && touch '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'" +} + +scenario_fix() { + mfix coredump_clean +} + +scenario_assert() { + mexec "test ! -e '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && test -e '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'" +} diff --git a/scripts/testing/maint-scenarios/20-cache-keep3.sh b/scripts/testing/maint-scenarios/20-cache-keep3.sh new file mode 100644 index 0000000..a9a6ee5 --- /dev/null +++ b/scripts/testing/maint-scenarios/20-cache-keep3.sh @@ -0,0 +1,24 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: paccache -rk3 trims an installed package's cache to 3 versions. +# +# paccache decides from filenames, so five fake versions of a package that +# IS installed (bash) stand in for a fat cache. The bootstrap may leave a +# real bash package in the cache too, so the assert uses the ordering +# invariant rather than a count: the two newest fakes survive any keep-3 +# window that contains them, and the two oldest fall outside it whether or +# not a real (newer) version is cached alongside. +SCENARIO_DESC="pacman cache trimmed to three versions per package" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "for v in 1.0.1 1.0.2 1.0.3 1.0.4 1.0.5; do touch /var/cache/pacman/pkg/bash-\$v-1-x86_64.pkg.tar.zst; done; ls /var/cache/pacman/pkg/ | grep -c '^bash-'" +} + +scenario_fix() { + mfix cache_clean +} + +scenario_assert() { + mexec "test -e /var/cache/pacman/pkg/bash-1.0.5-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-1.0.4-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.2-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.1-1-x86_64.pkg.tar.zst" +} diff --git a/scripts/testing/maint-scenarios/21-cache-uninstalled.sh b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh new file mode 100644 index 0000000..ea788e8 --- /dev/null +++ b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: paccache -ruk0 drops every cached version of uninstalled packages. +# +# Fake versions of a package that is NOT installed disappear entirely, while +# an installed package's cached version survives the uninstalled-only pass. +SCENARIO_DESC="cached versions of uninstalled packages dropped" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "touch /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst" +} + +scenario_fix() { + mfix cache_clean_uninstalled +} + +scenario_assert() { + mexec "test ! -e /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst" +} diff --git a/scripts/testing/maint-scenarios/22-orphan-remove.sh b/scripts/testing/maint-scenarios/22-orphan-remove.sh new file mode 100644 index 0000000..8412128 --- /dev/null +++ b/scripts/testing/maint-scenarios/22-orphan-remove.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a real orphan is created, detected, and removed by name. +# +# Installing `tree` --asdeps with no dependents makes it a true orphan +# (pacman -Qtdq lists it). The remedy is item-based: maint fix orphan_remove +# tree. Assert covers both the package state and the metric verdict. +SCENARIO_DESC="orphan package removed via the item-based remedy" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "pacman -S --noconfirm --asdeps tree && pacman -Qtdq | grep -qx tree" +} + +scenario_fix() { + mfix orphan_remove tree +} + +scenario_assert() { + mexec "! pacman -Qi tree >/dev/null 2>&1 && ! pacman -Qtdq | grep -qx tree" \ + && massert_metric pkg_orphans ok +} diff --git a/scripts/testing/maint-scenarios/23-pacnew-delete.sh b/scripts/testing/maint-scenarios/23-pacnew-delete.sh new file mode 100644 index 0000000..23cd8d1 --- /dev/null +++ b/scripts/testing/maint-scenarios/23-pacnew-delete.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a .pacnew file is deleted through the validated priv verb. +# +# pacdiff --output finds the planted file; the remedy takes the path as its +# item and rm's it. A neighbouring non-pacnew file proves the verb's path +# validation keeps the blast radius to the named file. +SCENARIO_DESC="planted .pacnew removed by path" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "cp /etc/pacman.conf /etc/pacman.conf.pacnew && pacdiff --output | grep -q pacman.conf.pacnew" +} + +scenario_fix() { + mfix pacnew_delete /etc/pacman.conf.pacnew +} + +scenario_assert() { + mexec "test ! -e /etc/pacman.conf.pacnew && test -e /etc/pacman.conf" \ + && massert_metric pkg_pacnew ok +} diff --git a/scripts/testing/maint-scenarios/30-unit-reset.sh b/scripts/testing/maint-scenarios/30-unit-reset.sh new file mode 100644 index 0000000..07eed53 --- /dev/null +++ b/scripts/testing/maint-scenarios/30-unit-reset.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a failed unit's state is reset. +# +# A transient oneshot that exits 1 lands in systemctl --failed; the remedy +# resets it and the failed list returns to its pre-break size. The unit name +# rides the remedy's item argument, exercising the priv verb's unit-name +# validation. The base image carries its own failed unit (grub-btrfsd — no +# snapshot dirs in the VM), so the assert scopes to this scenario's unit and +# the count delta, never the absolute failed list. +SCENARIO_DESC="failed unit reset clears the failed list" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="any" + +scenario_break() { + PRE_FAILED_COUNT=$(mexec "systemctl --failed --no-legend | wc -l") \ + && mexec "systemd-run --unit=maint-scenario-fail --service-type=oneshot --no-block /bin/false; sleep 2; systemctl is-failed maint-scenario-fail.service" +} + +scenario_fix() { + mfix unit_reset maint-scenario-fail.service +} + +scenario_assert() { + local pre="$PRE_FAILED_COUNT" + unset PRE_FAILED_COUNT # don't leak scenario state past this scenario + mexec "! systemctl is-failed maint-scenario-fail.service >/dev/null 2>&1" \ + && [ "$(mexec "systemctl --failed --no-legend | wc -l")" = "$pre" ] +} diff --git a/scripts/testing/maint-scenarios/31-fstrim-enable.sh b/scripts/testing/maint-scenarios/31-fstrim-enable.sh new file mode 100644 index 0000000..2d548a1 --- /dev/null +++ b/scripts/testing/maint-scenarios/31-fstrim-enable.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: fstrim.timer enabled and started from the disabled default. +# +# The base image ships util-linux with fstrim.timer disabled — the exact +# state the metric flags. The remedy is systemctl enable --now. +SCENARIO_DESC="fstrim.timer enabled and started" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="btrfs" + +scenario_break() { + # Default state, but pin it so the scenario stays honest if a future + # base image enables the timer. + mexec "systemctl disable --now fstrim.timer >/dev/null 2>&1 || true; ! systemctl is-enabled fstrim.timer >/dev/null 2>&1" +} + +scenario_fix() { + mfix fstrim_enable +} + +scenario_assert() { + mexec "systemctl is-enabled fstrim.timer && systemctl is-active fstrim.timer" +} diff --git a/scripts/testing/maint-scenarios/32-cron-enable.sh b/scripts/testing/maint-scenarios/32-cron-enable.sh new file mode 100644 index 0000000..39d4de6 --- /dev/null +++ b/scripts/testing/maint-scenarios/32-cron-enable.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: cronie enabled and started (the stopped-cron posture). +# +# Bootstrap installs cronie but never enables it — the broken state is the +# package's default. The remedy enables + starts the service. +SCENARIO_DESC="cronie enabled and started from the stopped state" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "systemctl disable --now cronie.service >/dev/null 2>&1 || true; ! systemctl is-active cronie.service >/dev/null 2>&1" +} + +scenario_fix() { + mfix cron_enable +} + +scenario_assert() { + mexec "systemctl is-enabled cronie.service && systemctl is-active cronie.service" +} diff --git a/scripts/testing/run-maint-nspawn.sh b/scripts/testing/run-maint-nspawn.sh new file mode 100644 index 0000000..04d4a32 --- /dev/null +++ b/scripts/testing/run-maint-nspawn.sh @@ -0,0 +1,268 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# Run pacman-level maint remedy scenarios in a systemd-nspawn container +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# The fast lane for the maint scenario suite: pure pacman-level cases (the +# "packages" scenario group — cache trims, orphan removal, pacnew deletion) +# don't need a booted machine, so they run in seconds against a pacstrap'd +# rootfs instead of minutes against the VM. Everything else (systemd, logs) +# needs pid 1 and journald — that stays in run-maint-scenarios.sh. +# +# The scenario files are shared with the VM runner and see the same helpers +# (mexec/mmaint/mfix/massert_metric); only the transport differs — the +# helper trio is deliberately duplicated between the two runners rather +# than abstracted over ssh-vs-nspawn. +# +# The rootfs is cached at $NSPAWN_ROOT and rebuilt with --fresh. Building +# and running need root (pacstrap, systemd-nspawn) via sudo. +# +# Usage: run-maint-nspawn.sh [--list] [--fresh] +# --list print the validated scenario plan and exit (no root needed) +# --fresh delete the cached rootfs and pacstrap a new one +# Env: MAINT_SCENARIO_DIR scenario dir (default: maint-scenarios/) +# MAINT_SRC maint package source tree +# (default: ~/.dotfiles/maint/src/maint) +# NSPAWN_ROOT rootfs dir (default: /var/tmp/archsetup-maint-nspawn) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" + +SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" +MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" +THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml" +NSPAWN_ROOT="${NSPAWN_ROOT:-/var/tmp/archsetup-maint-nspawn}" +NSPAWN_GROUP="packages" + +LIST_ONLY=false +FRESH=false + +usage() { + echo "Usage: $0 [--list] [--fresh]" + echo " --list print the validated scenario plan and exit" + echo " --fresh rebuild the cached rootfs from scratch" + echo "Env: MAINT_SCENARIO_DIR, MAINT_SRC, NSPAWN_ROOT" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --list) LIST_ONLY=true; shift ;; + --fresh) FRESH=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# ─── Plan: the pacman-level subset ─────────────────────────────────── +# Same contract validation as the VM runner; the selection rule is the +# group name — "packages" is pacman-level by construction. + +S_FILES=() S_NAMES=() S_DESCS=() + +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +build_plan() { + [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" + local f base + for f in "$SCENARIO_DIR"/*.sh; do + [ -e "$f" ] || break + _validate_scenario "$f" + [ "$(_scenario_var "$f" SCENARIO_GROUP)" = "$NSPAWN_GROUP" ] || continue + base=$(basename "$f" .sh) + S_FILES+=("$f") + S_NAMES+=("${base#[0-9][0-9]-}") + S_DESCS+=("$(_scenario_var "$f" SCENARIO_DESC)") + done + [ ${#S_FILES[@]} -gt 0 ] \ + || fatal "no '$NSPAWN_GROUP'-group scenarios in $SCENARIO_DIR" +} + +print_plan() { + local i + echo "maint nspawn scenario plan (group: $NSPAWN_GROUP)" + for i in "${!S_FILES[@]}"; do + echo " ${S_NAMES[$i]} — ${S_DESCS[$i]}" + done +} + +if [ "$LIST_ONLY" = "true" ]; then + LOGFILE=/dev/null + build_plan + print_plan + exit 0 +fi + +# ─── Run ───────────────────────────────────────────────────────────── + +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +RESULTS_DIR="$PROJECT_ROOT/test-results/maint-nspawn-$TIMESTAMP" +mkdir -p "$RESULTS_DIR" +LOGFILE="$RESULTS_DIR/scenarios.log" +init_logging "$LOGFILE" + +build_plan + +section "Maint Remedy Scenarios (nspawn): $TIMESTAMP" +info "Rootfs: $NSPAWN_ROOT" +info "Maint source: $MAINT_SRC" +print_plan | tee -a "$LOGFILE" + +command -v pacstrap >/dev/null \ + || fatal "pacstrap not found — pacman -S arch-install-scripts" +command -v systemd-nspawn >/dev/null || fatal "systemd-nspawn not found" +[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC" +[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML" +sudo -n true 2>/dev/null || fatal "needs passwordless sudo (pacstrap/nspawn)" + +# ─── Target helpers (available to scenario scripts) ────────────────── + +mexec() { # run a command in the container as root + sudo systemd-nspawn -q -D "$NSPAWN_ROOT" --pipe \ + /bin/bash -c "$*" 2>> "$LOGFILE" +} + +mmaint() { + mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*" +} + +mfix() { + local id="$1"; shift || true + step "maint fix $id ${*:+$* }--dry-run" + mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \ + || { error "dry-run failed: maint fix $id $*"; return 1; } + step "maint fix $id $*" + mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE" + return "${PIPESTATUS[0]}" +} + +massert_metric() { # <metric-id> <expected-severity> + local mid="$1" want="$2" got + got=$(mmaint status --json | python3 -c " +import json, sys +env = json.load(sys.stdin) +ms = [m for m in env['metrics'] if m['id'] == '$mid'] +print(ms[0]['severity'] if ms else 'MISSING') +") + if [ "$got" = "$want" ]; then + success "metric $mid is $want" + return 0 + fi + error "metric $mid: expected $want, got $got" + return 1 +} + +# ─── Rootfs ────────────────────────────────────────────────────────── + +if [ "$FRESH" = "true" ] && [ -d "$NSPAWN_ROOT" ]; then + step "Removing cached rootfs (--fresh)" + sudo rm -rf "$NSPAWN_ROOT" +fi + +if [ ! -x "$NSPAWN_ROOT/usr/bin/python3" ]; then + section "Building rootfs (pacstrap)" + sudo mkdir -p "$NSPAWN_ROOT" + # shellcheck disable=SC2024 # the log is user-owned by design + sudo pacstrap -c -K "$NSPAWN_ROOT" base python pacman-contrib expac \ + >> "$LOGFILE" 2>&1 || fatal "pacstrap failed (see $LOGFILE)" + success "Rootfs built" +else + info "Reusing cached rootfs (--fresh to rebuild)" +fi + +section "Installing maint tree + thresholds into the container" +sudo rm -rf "$NSPAWN_ROOT/root/maint-pkg" +sudo mkdir -p "$NSPAWN_ROOT/root/maint-pkg" "$NSPAWN_ROOT/root/.config/archsetup" +sudo cp -r "$MAINT_SRC" "$NSPAWN_ROOT/root/maint-pkg/" +sudo find "$NSPAWN_ROOT/root/maint-pkg" -name '__pycache__' -type d \ + -exec rm -rf {} + 2>/dev/null || true +sudo cp "$THRESHOLDS_TOML" \ + "$NSPAWN_ROOT/root/.config/archsetup/maintenance-thresholds.toml" + +step "Smoke: maint status --json runs in the container" +mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \ + || fatal "maint status failed in the container" +success "Container ready" + +# ─── Execution ─────────────────────────────────────────────────────── + +PASS=() FAIL=() + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} + +for i in "${!S_FILES[@]}"; do + run_scenario "$i" +done + +# ─── Report ────────────────────────────────────────────────────────── + +section "Results" +for n in "${PASS[@]}"; do success "PASS $n"; done +for n in "${FAIL[@]}"; do error "FAIL $n"; done +info "Log: $LOGFILE" +info "Rootfs kept for reuse: $NSPAWN_ROOT (--fresh to rebuild)" + +if [ ${#FAIL[@]} -gt 0 ]; then + error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed" + exit 1 +fi +success "All ${#PASS[@]} scenarios passed" +exit 0 diff --git a/scripts/testing/run-maint-scenarios.sh b/scripts/testing/run-maint-scenarios.sh new file mode 100644 index 0000000..448f6d9 --- /dev/null +++ b/scripts/testing/run-maint-scenarios.sh @@ -0,0 +1,374 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# Run maintenance-console remedy scenarios in the test VM +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# Layer 3 of the maint test strategy: each scenario breaks the VM in a known +# way over ssh, runs the real `maint fix <id>` inside the VM, and asserts the +# post-state. Scenarios live in maint-scenarios/*.sh and declare a GROUP; +# scenarios in one group are non-conflicting and share a VM boot, and a +# stop -> restore -> boot cycle runs only between groups (snapshot ops need +# the VM stopped). A scenario that fails may leave residue behind for its +# in-group successors -- the group restore boundary contains it, so read a +# multi-failure group front to back. +# +# Boot plan: +# 1. restore clean-install, boot, bootstrap (packages + maint tree + TOML) +# 2. stop, snapshot maint-ready +# 3. per group: boot, run scenarios, stop, restore maint-ready +# 4. restore clean-install, delete maint-ready (base stays pristine) +# +# Usage: run-maint-scenarios.sh [--list] [--group NAME] [--keep] +# --list print the validated batch plan and exit (no VM, no KVM needed) +# --group run (or list) only the named group +# --keep keep the VM running in its post-run state; skip the restores +# Env: FS_PROFILE=btrfs|zfs base image + scenario profile filter +# MAINT_SCENARIO_DIR scenario dir (default: maint-scenarios/) +# MAINT_SRC maint package source tree +# (default: ~/.dotfiles/maint/src/maint) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" + +SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" +MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" +THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml" +ROOT_PASSWORD="archsetup" +READY_SNAPSHOT="maint-ready" + +LIST_ONLY=false +ONLY_GROUP="" +KEEP_VM=false + +usage() { + echo "Usage: $0 [--list] [--group NAME] [--keep]" + echo " --list print the validated scenario plan and exit (no VM)" + echo " --group run only the named scenario group" + echo " --keep keep the VM running after the run (skip restores)" + echo "Env: FS_PROFILE=btrfs|zfs, MAINT_SCENARIO_DIR, MAINT_SRC" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --list) LIST_ONLY=true; shift ;; + --group) ONLY_GROUP="${2:?--group requires a value}"; shift 2 ;; + --keep) KEEP_VM=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# ─── Plan: enumerate, validate, group ──────────────────────────────── +# The plan layer is pure -- no VM, no KVM -- so --list works anywhere and +# the unit suite (tests/maint-scenarios/) exercises it directly. + +# Parallel arrays over scenarios, in file order. +S_FILES=() S_NAMES=() S_DESCS=() S_GROUPS=() +# Groups in first-appearance order. +GROUP_ORDER=() + +# Extract one declared var from a scenario file by sourcing it in a clean +# subshell. Scenario files must only define vars + functions; any top-level +# command fails here (no helpers exist), which is the contract. +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +_profile_matches() { # <profiles> -> 0 if scenario runs under $FS_PROFILE + local p + for p in $1; do + [ "$p" = "any" ] || [ "$p" = "$FS_PROFILE" ] && return 0 + done + return 1 +} + +build_plan() { + [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" + local f base name desc group profiles seen_groups=" " + local any=false + for f in "$SCENARIO_DIR"/*.sh; do + [ -e "$f" ] || break + any=true + _validate_scenario "$f" + base=$(basename "$f" .sh) + name="${base#[0-9][0-9]-}" + desc=$(_scenario_var "$f" SCENARIO_DESC) + group=$(_scenario_var "$f" SCENARIO_GROUP) + profiles=$(_scenario_var "$f" SCENARIO_PROFILES) + _profile_matches "$profiles" || continue + [ -n "$ONLY_GROUP" ] && [ "$group" != "$ONLY_GROUP" ] && continue + S_FILES+=("$f"); S_NAMES+=("$name") + S_DESCS+=("$desc"); S_GROUPS+=("$group") + case "$seen_groups" in + *" $group "*) ;; + *) GROUP_ORDER+=("$group"); seen_groups="$seen_groups$group " ;; + esac + done + [ "$any" = "true" ] || fatal "no scenario files in $SCENARIO_DIR" + if [ -n "$ONLY_GROUP" ] && [ ${#S_FILES[@]} -eq 0 ]; then + fatal "no scenarios match group '$ONLY_GROUP' (profile $FS_PROFILE)" + fi + [ ${#S_FILES[@]} -gt 0 ] \ + || fatal "no scenarios match profile $FS_PROFILE" +} + +print_plan() { + local g i + echo "maint scenario plan (profile: $FS_PROFILE)" + for g in "${GROUP_ORDER[@]}"; do + echo "group $g:" + for i in "${!S_FILES[@]}"; do + [ "${S_GROUPS[$i]}" = "$g" ] \ + && echo " ${S_NAMES[$i]} — ${S_DESCS[$i]}" + done + done +} + +# --list needs no logging file, VM, or KVM. Give fatal() a LOGFILE target +# only when we actually run. +if [ "$LIST_ONLY" = "true" ]; then + LOGFILE=/dev/null + build_plan + print_plan + exit 0 +fi + +# ─── Run ───────────────────────────────────────────────────────────── + +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +RESULTS_DIR="$PROJECT_ROOT/test-results/maint-$TIMESTAMP" +mkdir -p "$RESULTS_DIR" +LOGFILE="$RESULTS_DIR/scenarios.log" +init_logging "$LOGFILE" +init_vm_paths "$PROJECT_ROOT/vm-images" + +build_plan + +section "Maint Remedy Scenarios: $TIMESTAMP" +info "Profile: $FS_PROFILE (image: $(basename "$DISK_PATH"))" +info "Maint source: $MAINT_SRC" +print_plan | tee -a "$LOGFILE" + +[ -f "$DISK_PATH" ] || fatal "base disk not found: $DISK_PATH — build it: FS_PROFILE=$FS_PROFILE $SCRIPT_DIR/create-base-vm.sh" +snapshot_exists "$DISK_PATH" "clean-install" \ + || fatal "snapshot 'clean-install' not found on $DISK_PATH" +[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC" +[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML" +check_prerequisites || fatal "missing prerequisites" + +CLEANUP_DONE=0 +cleanup_scenarios() { + [ "$CLEANUP_DONE" = "1" ] && return 0 + CLEANUP_DONE=1 + [ "$KEEP_VM" = "true" ] && return 0 + # Never silent: a failed restore/delete here leaves the base image dirty + # for the next `make test`, so name it even though the trap can't abort. + stop_qemu 2>/dev/null \ + || echo "[!] cleanup: stop_qemu failed — VM may still hold the image" >&2 + restore_snapshot "$DISK_PATH" "clean-install" 2>/dev/null \ + || echo "[!] cleanup: clean-install restore FAILED — base image is dirty" >&2 + if snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT"; then + delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" 2>/dev/null \ + || echo "[!] cleanup: stray '$READY_SNAPSHOT' snapshot left on the base image" >&2 + fi +} +trap cleanup_scenarios EXIT + +# ─── Target helpers (available to scenario scripts) ────────────────── + +mexec() { # run a command in the VM as root + vm_exec "$ROOT_PASSWORD" "$@" +} + +# maint invocation inside the VM. Root runs the verbs directly, so +# MAINT_SUDO is empty (same switch the unit suites use). +mmaint() { + mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*" +} + +# Run one remedy: dry-run first (argv parity gate — the printed argv is the +# arm-press string), then for real. Fails if either exits non-zero. +mfix() { + local id="$1"; shift || true + step "maint fix $id ${*:+$* }--dry-run" + mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \ + || { error "dry-run failed: maint fix $id $*"; return 1; } + step "maint fix $id $*" + mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE" + return "${PIPESTATUS[0]}" +} + +# Assert a metric's severity in `maint status --json`. +massert_metric() { # <metric-id> <expected-severity> + local mid="$1" want="$2" got + got=$(mmaint status --json | python3 -c " +import json, sys +env = json.load(sys.stdin) +ms = [m for m in env['metrics'] if m['id'] == '$mid'] +print(ms[0]['severity'] if ms else 'MISSING') +") + if [ "$got" = "$want" ]; then + success "metric $mid is $want" + return 0 + fi + error "metric $mid: expected $want, got $got" + return 1 +} + +# ─── Bootstrap ─────────────────────────────────────────────────────── + +boot_vm() { + start_qemu "$DISK_PATH" "disk" "" "none" || fatal "failed to start VM" + wait_for_ssh "$ROOT_PASSWORD" 180 || fatal "VM SSH not available" +} + +bootstrap_vm() { + section "Bootstrap: packages + maint tree + thresholds" + step "Refreshing keyring + installing runtime deps" + # The base image can be months old: refresh the keyring first so the + # dep install doesn't die on expired signatures. + mexec "pacman -Sy --noconfirm archlinux-keyring" >> "$LOGFILE" 2>&1 \ + || fatal "keyring refresh failed (network up?)" + mexec "pacman -S --noconfirm python pacman-contrib expac cronie" \ + >> "$LOGFILE" 2>&1 || fatal "dep install failed" + + step "Copying maint package tree" + local tarball + tarball=$(mktemp) + tar -C "$(dirname "$MAINT_SRC")" --exclude='__pycache__' -czf "$tarball" \ + "$(basename "$MAINT_SRC")" + copy_to_vm "$tarball" "/tmp/maint-src.tgz" "$ROOT_PASSWORD" + rm -f "$tarball" + mexec "rm -rf /root/maint-pkg && mkdir -p /root/maint-pkg && tar -C /root/maint-pkg -xzf /tmp/maint-src.tgz && rm /tmp/maint-src.tgz" \ + >> "$LOGFILE" 2>&1 || fatal "maint tree extract failed" + + step "Installing thresholds TOML" + copy_to_vm "$THRESHOLDS_TOML" "/tmp/maintenance-thresholds.toml" \ + "$ROOT_PASSWORD" + mexec "mkdir -p /root/.config/archsetup && mv /tmp/maintenance-thresholds.toml /root/.config/archsetup/maintenance-thresholds.toml" \ + >> "$LOGFILE" 2>&1 || fatal "TOML install failed" + + step "Smoke: maint status --json runs" + mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \ + || fatal "maint status failed in the VM" + success "Bootstrap complete ($(python3 -c " +import json; print(len(json.load(open('$RESULTS_DIR/bootstrap-status.json'))['metrics']))") metrics probed)" +} + +# ─── Execution ─────────────────────────────────────────────────────── + +PASS=() FAIL=() + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # Sourced in the runner's shell so break/fix/assert see the helpers; + # unset afterwards so the next scenario can't inherit stale functions. + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} + +section "Preparing VM" +stop_qemu 2>/dev/null || true +step "Restoring clean-install snapshot" +restore_snapshot "$DISK_PATH" "clean-install" || fatal "restore failed" +boot_vm +bootstrap_vm + +step "Freezing bootstrap as snapshot: $READY_SNAPSHOT" +stop_qemu +snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT" \ + && delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" +create_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "snapshot failed" + +first_group=true +for g in "${GROUP_ORDER[@]}"; do + section "Group: $g" + if [ "$first_group" = "true" ]; then + first_group=false + else + step "Isolation boundary: restore $READY_SNAPSHOT" + stop_qemu + restore_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "restore failed" + fi + boot_vm + for i in "${!S_FILES[@]}"; do + [ "${S_GROUPS[$i]}" = "$g" ] && run_scenario "$i" + done + stop_qemu +done + +# ─── Report ────────────────────────────────────────────────────────── + +section "Results" +for n in "${PASS[@]}"; do success "PASS $n"; done +for n in "${FAIL[@]}"; do error "FAIL $n"; done +info "Log: $LOGFILE" + +if [ "$KEEP_VM" = "true" ]; then + warn "--keep: VM left in post-run state; base image NOT restored" + warn "restore by hand: qemu-img snapshot -a clean-install $DISK_PATH" + CLEANUP_DONE=1 +else + cleanup_scenarios +fi + +if [ ${#FAIL[@]} -gt 0 ]; then + error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed" + exit 1 +fi +success "All ${#PASS[@]} scenarios passed" +exit 0 diff --git a/scripts/testing/tests/test_boot.py b/scripts/testing/tests/test_boot.py index 78b4404..e442682 100644 --- a/scripts/testing/tests/test_boot.py +++ b/scripts/testing/tests/test_boot.py @@ -65,3 +65,19 @@ def test_zfs_has_sanoid(host): if not host.exists("zfs"): pytest.skip("ZFS not installed (non-ZFS system)") assert host.exists("sanoid"), "ZFS system should have sanoid installed" + + +def test_zfs_pre_pacman_snapshot_hook(host): + # archsetup installs a PreTransaction pacman hook + a self-pruning script so + # every pacman transaction is preceded by a rollback snapshot (configure_ + # pre_pacman_snapshots, run late in boot_ux). ZFS-root only. + if not host.exists("zfs"): + pytest.skip("ZFS not installed (non-ZFS system)") + script = host.file("/usr/local/bin/zfs-pre-snapshot") + assert script.exists and script.is_file, "pre-pacman snapshot script missing" + assert script.mode & 0o111, "pre-pacman snapshot script is not executable" + hook = host.file("/etc/pacman.d/hooks/zfs-snapshot.hook") + assert hook.exists and hook.is_file, "zfs-snapshot.hook missing" + assert "PreTransaction" in hook.content_string, "hook not PreTransaction" + assert "/usr/local/bin/zfs-pre-snapshot" in hook.content_string, \ + "hook does not exec the snapshot script" diff --git a/scripts/testing/tests/test_desktop.py b/scripts/testing/tests/test_desktop.py index 53e54e1..6f79bfd 100644 --- a/scripts/testing/tests/test_desktop.py +++ b/scripts/testing/tests/test_desktop.py @@ -50,6 +50,19 @@ def test_hyprland_config_present(host, hyprland_installed, home, rel): @pytest.mark.attribution("archsetup") +def test_live_update_guard_installed(host, hyprland_installed): + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + guard = host.file("/usr/local/bin/hypr-live-update-guard") + assert guard.exists, "live-update guard script missing" + assert guard.mode & 0o111, "live-update guard not executable" + hook = host.file("/etc/pacman.d/hooks/hypr-live-update-guard.hook") + assert hook.exists, "live-update guard pacman hook missing" + assert "hypr-live-update-guard" in hook.content_string, \ + "hook does not invoke the guard script" + + +@pytest.mark.attribution("archsetup") def test_portal_settings_backend_not_disabled(host, hyprland_installed, home): if not hyprland_installed: pytest.skip("Hyprland not installed") @@ -96,3 +109,32 @@ def test_autologin_configured(host): if not conf.exists: pytest.skip("autologin not configured (AUTOLOGIN=no, may be intentional)") assert conf.exists + + +BT_PANEL_BINS = ["bt", "bt-panel", "bt-priv", "waybar-bt"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("name", BT_PANEL_BINS) +def test_bt_panel_bin_stowed(host, hyprland_installed, home, name): + # Executable via either stow shape (per-file symlink or folded dir). + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + path = "%s/.local/bin/%s" % (home, name) + assert host.file(path).exists, "%s missing from ~/.local/bin" % name + assert host.run("test -x %s" % path).rc == 0, "%s not executable" % name + + +@pytest.mark.attribution("archsetup") +def test_bt_panel_wired(host, hyprland_installed, home): + # A fresh install lands the panel reachable: bar module, keybind, css. + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + waybar = host.file("%s/.config/waybar/config" % home) + assert "custom/bluetooth" in waybar.content_string, \ + "waybar config lacks the custom/bluetooth module" + hyprconf = host.file("%s/.config/hypr/hyprland.conf" % home) + assert "bt-panel" in hyprconf.content_string, \ + "hyprland.conf lacks the bt-panel keybind" + assert host.file("%s/.config/themes/dupre/panel.css" % home).exists, \ + "shared panel css missing from the stowed theme" diff --git a/scripts/testing/tests/test_packages.py b/scripts/testing/tests/test_packages.py index f237088..e0387d6 100644 --- a/scripts/testing/tests/test_packages.py +++ b/scripts/testing/tests/test_packages.py @@ -58,3 +58,52 @@ def test_git_installed(host): @pytest.mark.parametrize("tool", DEV_TOOLS) def test_dev_tool_present(host, tool): assert host.exists(tool), "dev tool %s missing from PATH" % tool + + +BLUETOOTH_STACK = ["bluez", "bluez-utils"] +VPN_STACK = ["wireguard-tools", "proton-vpn-cli", "tailscale"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", BLUETOOTH_STACK) +def test_bluetooth_stack_installed(host, pkg): + assert host.package(pkg).is_installed + + +# bt panel replaced blueman; zoom-web replaced zoom; the net panel's Tunnels +# view + proton-vpn-cli replaced the GTK app (they can't run concurrently). +RETIRED_PACKAGES = ["blueman", "zoom", "proton-vpn-gtk-app"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", RETIRED_PACKAGES) +def test_retired_package_not_installed(host, pkg): + # A reappearance means an install step regressed. + assert not host.package(pkg).is_installed + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", VPN_STACK) +def test_vpn_stack_installed(host, pkg): + assert host.package(pkg).is_installed + + +@pytest.mark.attribution("archsetup") +def test_tailscale_operator_granted(host, target_user): + # The installer grants operator so the net panel can toggle tailscale + # without sudo. Prefs only answer when the daemon is up. + if not host.service("tailscaled").is_running: + pytest.skip("tailscaled not running") + out = host.run("tailscale debug prefs") + assert out.rc == 0, "tailscale debug prefs failed" + assert '"OperatorUser": "%s"' % target_user in out.stdout + + +@pytest.mark.attribution("archsetup") +def test_eask_installed_user_local(host, home): + # Installed via npm -g --prefix ~/.local as the user; chime and + # linear-emacs shell out to it. + f = host.file("%s/.local/bin/eask" % home) + assert f.exists, "eask missing from ~/.local/bin" + npmrc = host.file("%s/.npmrc" % home) + assert npmrc.exists, ".npmrc (user npm prefix) not stowed" diff --git a/scripts/zfs-pre-snapshot b/scripts/zfs-pre-snapshot new file mode 100755 index 0000000..ed914d0 --- /dev/null +++ b/scripts/zfs-pre-snapshot @@ -0,0 +1,43 @@ +#!/bin/bash +# Snapshot the root dataset before a pacman transaction, then prune to the most +# recent $KEEP pre-pacman snapshots. Run from the zfs-snapshot.hook pacman hook +# (PreTransaction). Sanoid doesn't manage these (they aren't autosnap_ names), +# so retention is enforced here at creation time. +# +# Defaults match the live zroot layout; the ZFS_PRE_* env vars override them so +# the pruning logic is unit-testable against a fake zfs on PATH. + +POOL="${ZFS_PRE_POOL:-zroot}" +DATASET="${ZFS_PRE_DATASET:-$POOL/ROOT/default}" +LOCKFILE="${ZFS_PRE_LOCKFILE:-/tmp/.zfs-pre-snapshot.lock}" +MIN_INTERVAL="${ZFS_PRE_MIN_INTERVAL:-60}" +KEEP="${ZFS_PRE_KEEP:-10}" # pre-pacman snapshots to retain (recent-transaction rollback) + +# Skip if a snapshot was created within the last $MIN_INTERVAL seconds. A single +# pacman invocation can fire several transactions; this stops a burst of them +# from each cutting a near-identical snapshot. +if [ -f "$LOCKFILE" ]; then + last=$(stat -c %Y "$LOCKFILE" 2>/dev/null || echo 0) + now=$(date +%s) + if (( now - last < MIN_INTERVAL )); then + exit 0 + fi +fi + +TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S) +SNAPSHOT_NAME="pre-pacman_$TIMESTAMP" + +if zfs snapshot "$DATASET@$SNAPSHOT_NAME"; then + echo "Created snapshot: $DATASET@$SNAPSHOT_NAME" + touch "$LOCKFILE" + + # Keep only the most recent $KEEP pre-pacman snapshots; destroy older ones. + zfs list -H -o name -t snapshot -s creation "$DATASET" 2>/dev/null \ + | grep '@pre-pacman_' \ + | head -n -"$KEEP" \ + | while read -r old; do + zfs destroy "$old" && echo "Pruned old snapshot: $old" + done +else + echo "Warning: Failed to create snapshot" >&2 +fi |