diff options
Diffstat (limited to 'scripts/testing')
31 files changed, 2019 insertions, 904 deletions
diff --git a/scripts/testing/archsetup-test-zfs.conf b/scripts/testing/archsetup-test-zfs.conf new file mode 100644 index 0000000..a5459cf --- /dev/null +++ b/scripts/testing/archsetup-test-zfs.conf @@ -0,0 +1,21 @@ +# archsetup-test-zfs.conf - Archangel config for archsetup ZFS test VMs +# Used by create-base-vm.sh (FS_PROFILE=zfs) for fully automated base VM creation +# +# Usage: archangel --config-file /root/archsetup-test.conf +# +# Note: User creation is handled by archsetup, not archangel. +# See archsetup-vm.conf for archsetup configuration (shared across profiles - +# archsetup detects ZFS from the live root, so it needs no filesystem setting). +# +# Unencrypted ZFS root: encryption isn't what the harness validates, and +# NO_ENCRYPT=yes skips the passphrase prompt for a fully unattended install. + +FILESYSTEM=zfs +HOSTNAME=archsetup-test +TIMEZONE=America/Chicago +LOCALE=en_US.UTF-8 +KEYMAP=us +DISKS=/dev/vda +NO_ENCRYPT=yes +ROOT_PASSWORD=archsetup +ENABLE_SSH=yes diff --git a/scripts/testing/create-base-vm.sh b/scripts/testing/create-base-vm.sh index b8a4e2b..e626813 100755 --- a/scripts/testing/create-base-vm.sh +++ b/scripts/testing/create-base-vm.sh @@ -20,10 +20,19 @@ source "$SCRIPT_DIR/lib/vm-utils.sh" # Configuration VM_IMAGES_DIR="$PROJECT_ROOT/vm-images" -CONFIG_FILE="$SCRIPT_DIR/archsetup-test.conf" LIVE_ISO_PASSWORD="archangel" SNAPSHOT_NAME="clean-install" +# FS_PROFILE (btrfs default / zfs) picks the archangel base-install config. +# btrfs -> archsetup-test.conf, zfs -> archsetup-test-zfs.conf. The matching +# base image name is derived from FS_PROFILE by init_vm_paths. +FS_PROFILE="${FS_PROFILE:-btrfs}" +if [ "$FS_PROFILE" = "btrfs" ]; then + CONFIG_FILE="$SCRIPT_DIR/archsetup-test.conf" +else + CONFIG_FILE="$SCRIPT_DIR/archsetup-test-${FS_PROFILE}.conf" +fi + # Initialize logging mkdir -p "$PROJECT_ROOT/test-results" LOGFILE="$PROJECT_ROOT/test-results/create-base-vm-$(date +'%Y%m%d-%H%M%S').log" diff --git a/scripts/testing/lib/network-diagnostics.sh b/scripts/testing/lib/network-diagnostics.sh index 38788e5..dc54334 100644 --- a/scripts/testing/lib/network-diagnostics.sh +++ b/scripts/testing/lib/network-diagnostics.sh @@ -6,58 +6,110 @@ # Note: logging.sh and vm-utils.sh should already be sourced by the calling script # Uses globals: ROOT_PASSWORD, SSH_PORT, SSH_OPTS, VM_IP (from vm-utils.sh or calling script) +# Optional global: TEST_RESULTS_DIR (raw command outputs are saved there when set) -# Run quick network diagnostics +# Gather one read-only fact from the VM, print it, and save the raw output. +# Facts are collected regardless of pass/fail so a failing install still leaves +# the IP/route/resolver evidence in the log and the results dir. +# $1 label human-readable label for the fact +# $2 slug filename slug for the saved raw output +# $3 cmd remote command to run over the shared ssh_base +# Uses the caller's locals ssh_base and results_dir (dynamic scope). +_netdiag_fact() { + local label="$1" slug="$2" cmd="$3" out + out="$($ssh_base "$cmd" 2>&1)" + info "${label}:" + printf '%s\n' "$out" | while IFS= read -r line; do + info " $line" + done + if [ -n "$results_dir" ]; then + printf '%s\n' "$out" > "$results_dir/netdiag-${slug}.txt" 2>/dev/null || true + fi +} + +# Run quick network diagnostics. +# +# Evidence first: collect read-only facts (interfaces, route, resolver) +# unconditionally, then run every reachability check and report all failures at +# the end. A DNS failure is named as a DNS failure, not masked as a generic "no +# internet" or misattributed to the Arch mirror. Returns 0 when all checks pass, +# non-zero when any check fails, so callers keep their success/failure contract. run_network_diagnostics() { local password="${ROOT_PASSWORD:-archsetup}" local port="${SSH_PORT:-22}" local host="${VM_IP:-localhost}" local ssh_base="sshpass -p $password ssh $SSH_OPTS -p $port root@$host" + local results_dir="${TEST_RESULTS_DIR:-}" + local failures=() section "Pre-flight Network Diagnostics" - # Test 1: Basic connectivity (use curl instead of ping - SLIRP may not handle ICMP) - step "Testing internet connectivity" - if $ssh_base "curl -s --connect-timeout 5 -o /dev/null http://archlinux.org" 2>/dev/null; then - success "Internet connectivity OK" - else - error "No internet connectivity" - return 1 - fi + # --- Phase 1: collect read-only facts, unconditionally --- + # These never gate the outcome; they exist so a failed install still has + # the interface/route/resolver evidence to diagnose from. + step "Collecting interface addresses" + _netdiag_fact "Interface addresses (ip -brief addr)" "ip-addr" "ip -brief addr" + + step "Collecting default route" + _netdiag_fact "Default route (ip route show default)" "ip-route" "ip route show default" - # Test 2: DNS resolution (use getent which is always available, unlike nslookup/dig) + step "Reading resolver configuration" + _netdiag_fact "Resolver (/etc/resolv.conf)" "resolv-conf" "cat /etc/resolv.conf" + + # --- Phase 2: generic connectivity checks (run all, don't short-circuit) --- + # DNS, egress, and TLS are independent failure modes. Keeping them separate + # means a resolver problem reads as DNS, not as a downstream mirror failure. step "Testing DNS resolution" if $ssh_base "getent hosts archlinux.org >/dev/null 2>&1" 2>/dev/null; then success "DNS resolution OK" else error "DNS resolution failed" - return 1 + failures+=("DNS resolution (getent hosts archlinux.org)") fi - # Test 3: Arch mirror accessibility + step "Testing HTTP egress" + if $ssh_base "curl -s --connect-timeout 5 -o /dev/null http://archlinux.org" 2>/dev/null; then + success "HTTP egress OK" + else + error "HTTP egress failed" + failures+=("HTTP egress (http://archlinux.org)") + fi + + step "Testing TLS/HTTPS egress" + if $ssh_base "curl -s --connect-timeout 5 -o /dev/null https://archlinux.org" 2>/dev/null; then + success "TLS/HTTPS egress OK" + else + error "TLS/HTTPS egress failed" + failures+=("TLS/HTTPS egress (https://archlinux.org)") + fi + + # --- Phase 3: Arch-specific checks (run all, don't short-circuit) --- step "Testing Arch mirror access" if $ssh_base "curl -s -I https://geo.mirror.pkgbuild.com/ | head -1 | grep -qE '(200|301|302)'" 2>/dev/null; then success "Arch mirrors accessible" else error "Cannot reach Arch mirrors" - return 1 + failures+=("Arch mirror (https://geo.mirror.pkgbuild.com/)") fi - # Test 4: AUR accessibility step "Testing AUR access" if $ssh_base "curl -s -I https://aur.archlinux.org/ | head -1 | grep -qE '(200|405)'" 2>/dev/null; then success "AUR accessible" else error "Cannot reach AUR" - return 1 + failures+=("AUR (https://aur.archlinux.org/)") fi - # Show network info - info "Network configuration:" - $ssh_base "ip addr show | grep 'inet ' | grep -v '127.0.0.1'" 2>/dev/null | while IFS= read -r line; do - info " $line" - done + # --- Summary: report every failure, not just the first --- + if [ ${#failures[@]} -eq 0 ]; then + success "Network diagnostics complete - all checks passed" + return 0 + fi - success "Network diagnostics complete" - return 0 + error "Network diagnostics found ${#failures[@]} failure(s):" + local f + for f in "${failures[@]}"; do + error " - $f" + done + return 1 } diff --git a/scripts/testing/lib/testinfra.sh b/scripts/testing/lib/testinfra.sh new file mode 100644 index 0000000..0822a9f --- /dev/null +++ b/scripts/testing/lib/testinfra.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Testinfra post-install validation sweep (runs on the host, over SSH). +# +# This is the primary post-install validator (it replaced the shell +# run_all_validations sweep). It connects to the freshly-installed VM over SSH +# and runs the pytest suite under scripts/testing/tests/. Its result drives the +# run's pass/fail, and per-test failures are bucketed (archsetup / base_install +# / unknown) into the same issue-attribution report the shell sweep produced. +# +# Auth: reuse the root key the harness already authorized (inject_root_key), +# which survives the sshd prohibit-password hardening; mint our own only if the +# harness didn't (standalone use). pytest connects key-only via a generated +# ssh-config. Key + config live in the results dir and are discarded with it. +# +# Uses globals from run-test.sh / vm-utils.sh: SCRIPT_DIR, VM_IP, SSH_PORT, +# ROOT_PASSWORD, ROOT_SSH_KEY, ARCHSETUP_VM_CONF, plus the validation.sh +# helpers attribute_issue / VALIDATION_*. Toggle with RUN_TESTINFRA=false. + +# Record each pytest failure from the attribution file into the issue arrays +# (validation.sh's attribute_issue), so generate_issue_report covers them. +_testinfra_record_attribution() { + local file="$1" bucket="" + [ -f "$file" ] || return 0 + while IFS= read -r line; do + case "$line" in + "[archsetup]") bucket=archsetup ;; + "[base_install]") bucket=base ;; + "[unknown]") bucket=unknown ;; + " "*) attribute_issue "testinfra: ${line# }" "$bucket" ;; + esac + done < "$file" +} + +# run_testinfra_validation <results_dir> +# Returns 0 only when the pytest sweep ran and passed. Returns non-zero when it +# failed OR could not run (missing tooling / SSH setup) — a sweep that can't run +# is not a pass. RUN_TESTINFRA=false is the one explicit opt-out (returns 0). +run_testinfra_validation() { + local results_dir="$1" + local tests_dir="$SCRIPT_DIR/tests" + local key="$results_dir/testinfra_key" + local sshcfg="$results_dir/testinfra_ssh_config" + + if [ "${RUN_TESTINFRA:-true}" != "true" ]; then + warn "RUN_TESTINFRA=false - skipping the Testinfra validation sweep" + return 0 + fi + if ! command -v pytest >/dev/null 2>&1 || ! python3 -c 'import testinfra' >/dev/null 2>&1; then + error "Testinfra/pytest not installed on host - cannot validate (run: make deps)" + return 1 + fi + + section "Running Validation Checks (Testinfra)" + + # Prefer the harness's already-authorized root key; mint one if absent. + if [ -n "${ROOT_SSH_KEY:-}" ] && [ -f "${ROOT_SSH_KEY}" ]; then + key="$ROOT_SSH_KEY" + else + rm -f "$key" "$key.pub" + if ! ssh-keygen -t ed25519 -N "" -q -f "$key"; then + error "testinfra: ssh-keygen failed" + return 1 + fi + if ! copy_to_vm "$key.pub" "/tmp/testinfra_key.pub" "$ROOT_PASSWORD"; then + error "testinfra: pubkey copy failed" + return 1 + fi + if ! vm_exec "$ROOT_PASSWORD" \ + "mkdir -p /root/.ssh && chmod 700 /root/.ssh && cat /tmp/testinfra_key.pub >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"; then + error "testinfra: authorizing key in VM failed" + return 1 + fi + fi + + # ssh-config so testinfra connects key-only, no host-key prompt. + cat > "$sshcfg" <<EOF +Host testinfra-target + HostName ${VM_IP:-localhost} + Port ${SSH_PORT:-2222} + User root + IdentityFile $key + IdentitiesOnly yes + StrictHostKeyChecking no + UserKnownHostsFile /dev/null +EOF + + # The account archsetup created, for the tests that need it. + local test_user + test_user=$(sed -n 's/^USERNAME=//p' "$ARCHSETUP_VM_CONF" 2>/dev/null | head -n1) + : "${test_user:=cjennings}" + + local logf="$results_dir/testinfra.log" + ARCHSETUP_TEST_USER="$test_user" pytest "$tests_dir" \ + --hosts="ssh://testinfra-target" \ + --ssh-config="$sshcfg" \ + --attribution-file="$results_dir/testinfra-attribution.txt" \ + -v >> "$logf" 2>&1 + local rc=$? + + # Surface pytest's counts through the shared validation counters so the + # issue report summary is meaningful (the shell sweep no longer runs). + local summary + summary=$(grep -oE '[0-9]+ (passed|failed|error|errors|skipped)' "$logf" | tail -10) + VALIDATION_PASSED=$(echo "$summary" | awk '/passed/{print $1}' | tail -1); VALIDATION_PASSED=${VALIDATION_PASSED:-0} + VALIDATION_WARNINGS=$(echo "$summary" | awk '/skipped/{print $1}' | tail -1); VALIDATION_WARNINGS=${VALIDATION_WARNINGS:-0} + local nfail nerr + nfail=$(echo "$summary" | awk '/failed/{print $1}' | tail -1); nfail=${nfail:-0} + nerr=$(echo "$summary" | awk '/error/{print $1}' | tail -1); nerr=${nerr:-0} + VALIDATION_FAILED=$((nfail + nerr)) + + if [ "$rc" -eq 0 ]; then + success "Testinfra validation passed ($VALIDATION_PASSED passed, $VALIDATION_WARNINGS skipped)" + else + error "Testinfra validation failed ($VALIDATION_FAILED failed/error; see testinfra.log)" + _testinfra_record_attribution "$results_dir/testinfra-attribution.txt" + fi + return "$rc" +} diff --git a/scripts/testing/lib/validation.sh b/scripts/testing/lib/validation.sh index 6855da7..fa7ddcc 100644 --- a/scripts/testing/lib/validation.sh +++ b/scripts/testing/lib/validation.sh @@ -21,38 +21,7 @@ declare -a UNKNOWN_ISSUES # SSH helper (uses globals: VM_IP, ROOT_PASSWORD) ssh_cmd() { sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=10 -p "${SSH_PORT:-22}" "root@$VM_IP" "$@" 2>/dev/null -} - -# Validation result helpers -validation_pass() { - local test_name="$1" - success "$test_name" - ((VALIDATION_PASSED++)) || true -} - -validation_fail() { - local test_name="$1" - local details="${2:-}" - error "$test_name" - [ -n "$details" ] && info " Details: $details" - ((VALIDATION_FAILED++)) || true -} - -validation_warn() { - local test_name="$1" - local details="${2:-}" - warn "$test_name" - [ -n "$details" ] && info " Details: $details" - ((VALIDATION_WARNINGS++)) || true -} - -# A check whose precondition can't hold in this environment (headless VM, -# slirp networking, pre-reboot state). Logged for the record, counted nowhere -# — a warning that fires on every run trains readers to ignore warnings. -validation_skip() { - local test_name="$1" - info "SKIP: $test_name" + -o ConnectTimeout=10 ${SSH_KEY_OPT:-} -p "${SSH_PORT:-22}" "root@$VM_IP" "$@" 2>/dev/null } # Attribute an issue to archsetup or base install @@ -265,802 +234,6 @@ categorize_errors() { } #============================================================================= -# VALIDATION CHECKS -#============================================================================= - -run_all_validations() { - section "Running Validation Checks" - - # User & Authentication - validate_user_created - validate_user_shell - validate_user_groups - - # Dotfiles - validate_dotfiles - - # Package Managers - validate_yay_installed - validate_pacman_working - - # Window Manager (detects DWM or Hyprland automatically) - validate_window_manager - - # Essential Services - validate_firewall - validate_dns_config - validate_avahi - validate_fail2ban - validate_networkmanager - - # Developer Tools - validate_emacs - validate_git_config - validate_dev_tools - - # System Configuration - validate_zfs_config - validate_boot_config - validate_autologin_config - validate_gnome_keyring_setup - - # Boot & Initramfs (critical for ZFS systems) - validate_terminus_font - validate_mkinitcpio_hooks - validate_initramfs_consolefont - validate_nvme_module - - # Archsetup Specific - validate_archsetup_log - validate_state_markers -} - -#----------------------------------------------------------------------------- -# User & Authentication Validations -#----------------------------------------------------------------------------- - -validate_user_created() { - step "Checking if user 'cjennings' exists" - if ssh_cmd "id cjennings" &>> "$LOGFILE"; then - validation_pass "User cjennings exists" - else - validation_fail "User cjennings not found" - attribute_issue "User cjennings not created" "archsetup" - fi -} - -validate_user_shell() { - step "Checking if ZSH is default shell" - local shell=$(ssh_cmd "getent passwd cjennings | cut -d: -f7") - if [ "$shell" = "/bin/zsh" ] || [ "$shell" = "/usr/bin/zsh" ]; then - validation_pass "ZSH is default shell" - else - validation_fail "ZSH not default shell (got: $shell)" - attribute_issue "ZSH not set as default shell" "archsetup" - fi -} - -validate_user_groups() { - step "Checking user group memberships" - # Groups added by archsetup: - # - wheel (useradd -G wheel) - # - sys,adm,network,scanner,power,uucp,audio,lp,rfkill,video,storage,optical,users (usermod -aG) - # - docker (gpasswd -a, added later in developer_workstation) - local expected_groups="wheel sys adm network scanner power uucp audio lp rfkill video storage optical users docker" - local missing_groups="" - - for group in $expected_groups; do - if ! ssh_cmd "groups cjennings" | grep -q "\b$group\b"; then - missing_groups="$missing_groups $group" - fi - done - - if [ -z "$missing_groups" ]; then - validation_pass "User in all expected groups (15 groups)" - else - validation_fail "User missing groups:$missing_groups" - attribute_issue "User missing groups:$missing_groups" "archsetup" - fi -} - -#----------------------------------------------------------------------------- -# Dotfiles Validations -#----------------------------------------------------------------------------- - -validate_dotfiles() { - step "Checking dotfiles setup" - - # 1. Check if .zshrc is a symlink - if ! ssh_cmd "test -L /home/cjennings/.zshrc"; then - validation_fail "Dotfiles not stowed (.zshrc is not a symlink)" - attribute_issue "Dotfiles stow failed" "archsetup" - return 1 - fi - - # 2. Check symlink points to correct location. archsetup now clones the - # dotfiles repo to ~/.dotfiles and stows from there (DOTFILES_DIR default). - # Which tree owns .zshrc depends on DESKTOP_ENV: none stows the standalone - # minimal/ tree; dwm and hyprland stow common/. - local target=$(ssh_cmd "readlink /home/cjennings/.zshrc") - local desktop_env=$(sed -n 's/^DESKTOP_ENV=//p' "$ARCHSETUP_VM_CONF" 2>/dev/null | head -n1) - local expected_pattern=".dotfiles/common/.zshrc" - [ "$desktop_env" = "none" ] && expected_pattern=".dotfiles/minimal/.zshrc" - - if ! echo "$target" | grep -q "$expected_pattern"; then - validation_fail "Dotfiles symlink points to wrong location: $target" - attribute_issue "Dotfiles symlink incorrect: $target" "archsetup" - return 1 - fi - - # 3. Check the target file actually exists (not a broken symlink) - if ! ssh_cmd "test -f /home/cjennings/.zshrc"; then - validation_fail "Dotfiles symlink is broken (target doesn't exist)" - ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 - attribute_issue "Dotfiles symlink broken" "archsetup" - return 1 - fi - - # 4. Check user can actually read the file (not just root) - local result=$(ssh_cmd "sudo -u cjennings cat /home/cjennings/.zshrc > /dev/null 2>&1 && echo OK || echo FAIL") - if [ "$result" != "OK" ]; then - validation_fail "Dotfiles not readable by user (permission issue)" - ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 - attribute_issue "Dotfiles not readable by user" "archsetup" - return 1 - fi - - validation_pass "Dotfiles configured correctly (symlink to $target, readable by user)" -} - -#----------------------------------------------------------------------------- -# Package Manager Validations -#----------------------------------------------------------------------------- - -validate_yay_installed() { - step "Checking if yay (AUR helper) is installed and functional" - - # Check binary exists - if ! ssh_cmd "which yay" &>> "$LOGFILE"; then - validation_fail "yay not found" - attribute_issue "yay not installed" "archsetup" - return 1 - fi - - # Check yay can query packages (functional test) - if ssh_cmd "sudo -u cjennings yay -Qi yay" &>> "$LOGFILE"; then - validation_pass "yay is installed and functional" - else - validation_fail "yay binary exists but query failed" - attribute_issue "yay not functional" "archsetup" - fi -} - -validate_pacman_working() { - step "Checking if pacman is functional" - if ssh_cmd "pacman -Qi base" &>> "$LOGFILE"; then - validation_pass "pacman is functional" - else - validation_fail "pacman query failed" - attribute_issue "pacman not functional" "unknown" - fi -} - -#----------------------------------------------------------------------------- -# Window Manager Validations -#----------------------------------------------------------------------------- - -validate_suckless_tools() { - step "Checking suckless tools (dwm, st, dmenu, slock)" - local missing="" - - for tool in dwm st dmenu slock; do - if ! ssh_cmd "test -f /usr/local/bin/$tool"; then - missing="$missing $tool" - fi - done - - if [ -z "$missing" ]; then - validation_pass "All suckless tools installed (dwm, st, dmenu, slock)" - else - validation_fail "Missing suckless tools:$missing" - attribute_issue "Missing suckless tools:$missing" "archsetup" - fi -} - -validate_hyprland_tools() { - step "Checking Hyprland tools" - local missing="" - - # Check core Hyprland packages - for pkg in hyprland hypridle hyprlock waybar fuzzel swww grim slurp gammastep foot; do - if ! ssh_cmd "pacman -Q $pkg &>/dev/null"; then - missing="$missing $pkg" - fi - done - - if [ -z "$missing" ]; then - validation_pass "All Hyprland tools installed" - else - validation_fail "Missing Hyprland tools:$missing" - attribute_issue "Missing Hyprland tools:$missing" "archsetup" - fi -} - -validate_hyprland_config() { - step "Checking Hyprland configuration files" - local missing="" - - for config in ".config/hypr/hyprland.conf" ".config/hypr/hypridle.conf" \ - ".config/hypr/hyprlock.conf" ".config/waybar/config" \ - ".config/fuzzel/fuzzel.ini" ".config/gammastep/config.ini"; do - if ! ssh_cmd "test -f /home/cjennings/$config"; then - missing="$missing $config" - fi - done - - if [ -z "$missing" ]; then - validation_pass "All Hyprland config files present" - else - validation_fail "Missing Hyprland configs:$missing" - attribute_issue "Missing Hyprland configs:$missing" "archsetup" - fi -} - -validate_hyprland_socket() { - step "Checking Hyprland IPC socket" - # The socket only exists while the compositor runs. In the headless test - # VM nobody logs in graphically, so a missing socket with no Hyprland - # process is the expected state, not a finding. - if ssh_cmd "test -S /tmp/hypr/*/.socket.sock 2>/dev/null"; then - validation_pass "Hyprland socket exists" - elif ! ssh_cmd "pgrep -x Hyprland >/dev/null 2>&1"; then - validation_skip "Hyprland not running (headless) — socket check not applicable" - else - validation_warn "Hyprland running but IPC socket not found" - fi -} - -validate_portal_dark_mode() { - step "Checking Settings portal returns dark mode" - - # Check portals.conf exists and uses gtk for Settings - local portals_conf="/home/cjennings/.config/xdg-desktop-portal/portals.conf" - if ! ssh_cmd "test -f $portals_conf"; then - validation_fail "portals.conf not found" - attribute_issue "xdg-desktop-portal portals.conf missing" "archsetup" - return 1 - fi - - local settings_backend=$(ssh_cmd "grep 'org.freedesktop.impl.portal.Settings' $portals_conf 2>/dev/null | cut -d= -f2") - if [ "$settings_backend" = "none" ]; then - validation_fail "Settings portal disabled (set to 'none')" - attribute_issue "Settings portal disabled in portals.conf" "archsetup" - return 1 - fi - - # Query the portal for color-scheme (requires portal services running) - # Returns "v v u 1" for dark mode (1 = prefer-dark) - local color_scheme=$(ssh_cmd "sudo -u cjennings busctl --user call org.freedesktop.portal.Desktop /org/freedesktop/portal/desktop org.freedesktop.portal.Settings Read 'ss' 'org.freedesktop.appearance' 'color-scheme' 2>/dev/null | grep -o 'u [0-9]' | cut -d' ' -f2") - - if [ "$color_scheme" = "1" ]; then - validation_pass "Settings portal returns dark mode (color-scheme=1)" - elif [ -z "$color_scheme" ] && ! ssh_cmd "pgrep -x Hyprland >/dev/null 2>&1"; then - # No compositor → no graphical session bus to query. A socket-activated - # xdg-desktop-portal process can exist even headless, so the compositor - # is the real precondition (same condition as the socket check). The - # conf-file checks above already validated what install controls. - validation_skip "No compositor running (headless) — portal query not applicable" - elif [ -z "$color_scheme" ]; then - validation_warn "Could not query Settings portal (portal may not be running)" - else - validation_fail "Settings portal not returning dark mode (color-scheme=$color_scheme, expected 1)" - attribute_issue "Settings portal not configured for dark mode" "archsetup" - fi -} - -validate_window_manager() { - # Detect which desktop environment is installed and validate accordingly - if ssh_cmd "pacman -Q hyprland &>/dev/null"; then - section "Hyprland Desktop Environment" - validate_hyprland_tools - validate_hyprland_config - validate_hyprland_socket - validate_portal_dark_mode - elif ssh_cmd "test -f /usr/local/bin/dwm"; then - section "DWM Desktop Environment" - validate_suckless_tools - else - validation_warn "No window manager detected (DESKTOP_ENV=none?)" - fi -} - -#----------------------------------------------------------------------------- -# Essential Services Validations -#----------------------------------------------------------------------------- - -validate_firewall() { - step "Checking if firewall (ufw) is enabled" - local status=$(ssh_cmd "systemctl is-enabled ufw.service 2>/dev/null || echo disabled") - if [ "$status" = "enabled" ]; then - validation_pass "UFW firewall is enabled" - else - validation_fail "UFW firewall not enabled" - attribute_issue "UFW not enabled" "archsetup" - fi -} - -validate_dns_config() { - step "Checking DNS-over-TLS configuration" - if ssh_cmd "grep -q 'DNS=.*#' /etc/systemd/resolved.conf 2>/dev/null"; then - validation_pass "DNS-over-TLS configured" - else - validation_warn "DNS-over-TLS may not be configured" - fi -} - -validate_avahi() { - step "Checking avahi-daemon status" - local status=$(ssh_cmd "systemctl is-enabled avahi-daemon.service 2>/dev/null || echo disabled") - if [ "$status" = "enabled" ]; then - validation_pass "avahi-daemon is enabled" - - # Full-stack mDNS test: ping hostname.local. QEMU user-mode (slirp, - # 10.0.2.x) doesn't pass multicast, so mDNS genuinely can't resolve - # there — only run the ping on real networking. - if ssh_cmd "ip -4 addr show" 2>/dev/null | grep -q "10\.0\.2\."; then - validation_skip "mDNS ping not possible on slirp networking (no multicast)" - else - local hostname=$(ssh_cmd "hostname") - if ssh_cmd "ping -c 1 -W 2 ${hostname}.local" &>> "$LOGFILE"; then - validation_pass "mDNS working (${hostname}.local responds to ping)" - else - validation_warn "mDNS ping failed (avahi may need time to propagate)" - fi - fi - else - # This might be OK if avahi was pre-installed - validation_warn "avahi-daemon not enabled (may have been pre-configured)" - fi -} - -validate_fail2ban() { - step "Checking fail2ban status" - local status=$(ssh_cmd "systemctl is-enabled fail2ban.service 2>/dev/null || echo disabled") - if [ "$status" = "enabled" ]; then - validation_pass "fail2ban is enabled" - else - validation_fail "fail2ban not enabled" - attribute_issue "fail2ban not enabled" "archsetup" - fi -} - -validate_networkmanager() { - step "Checking NetworkManager status" - local status=$(ssh_cmd "systemctl is-enabled NetworkManager.service 2>/dev/null || echo disabled") - if [ "$status" = "enabled" ]; then - validation_pass "NetworkManager is enabled" - # Functional test - if ssh_cmd "nmcli general status" &>> "$LOGFILE"; then - validation_pass "NetworkManager is functional" - else - validation_warn "NetworkManager enabled but not responding" - fi - else - validation_fail "NetworkManager not enabled" - attribute_issue "NetworkManager not enabled" "archsetup" - fi -} - -#----------------------------------------------------------------------------- -# Service-Specific Validations -#----------------------------------------------------------------------------- - -validate_all_services() { - section "Service Validations" - - # Core services (always expected) - validate_service "sshd" "enabled" "active" - validate_service "systemd-resolved" "enabled" "active" - validate_service "ufw" "enabled" "" # VM lacks iptables modules, can't be active - validate_service "fail2ban" "enabled" "active" - validate_service "NetworkManager" "enabled" "active" - validate_service "rngd" "enabled" "active" - validate_service "cronie" "enabled" "" - validate_service "atd" "enabled" "" - - # Cron job: log cleanup - step "Checking log-cleanup cron job" - local crontab_entry=$(ssh_cmd "sudo -u cjennings crontab -l 2>/dev/null | grep log-cleanup") - if [ -n "$crontab_entry" ]; then - validation_pass "log-cleanup cron job installed" - else - validation_fail "log-cleanup cron job not in crontab" - attribute_issue "log-cleanup cron job missing from user crontab" "archsetup" - fi - - # Timer services - validate_service "reflector.timer" "enabled" "" - validate_service "paccache.timer" "enabled" "" - - # Optional services (warn if missing, don't fail) - validate_service_optional "avahi-daemon" "enabled" - validate_service_optional "bluetooth" "enabled" - validate_service_optional "cups" "enabled" - validate_service_optional "docker" "enabled" - validate_service_optional "tailscaled" "enabled" - # Syncthing uses user service (not system), check lingering is enabled. - # test -e, not ls: ls prints the path on success, so the old capture held - # "path\nyes" and could never equal "yes" — the check warned on every - # run, even with lingering correctly enabled. - step "Checking user lingering for syncthing" - local linger_enabled=$(ssh_cmd "test -e /var/lib/systemd/linger/cjennings && echo yes || echo no") - if [ "$linger_enabled" = "yes" ]; then - validation_pass "User lingering enabled for syncthing user service" - else - validation_warn "User lingering not enabled (syncthing may not autostart)" - fi - - # Filesystem-specific - validate_zfs_services - validate_btrfs_services - - # Functional tests - validate_service_functions -} - -validate_service() { - local service="$1" - local expected_enabled="$2" # "enabled" or "" - local expected_active="$3" # "active" or "" - - step "Checking $service" - - if [ -n "$expected_enabled" ]; then - local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled") - if [ "$enabled" = "enabled" ]; then - validation_pass "$service is enabled" - else - validation_fail "$service not enabled (got: $enabled)" - attribute_issue "$service not enabled" "archsetup" - return 1 - fi - fi - - if [ -n "$expected_active" ]; then - local active=$(ssh_cmd "systemctl is-active $service 2>/dev/null || echo inactive") - if [ "$active" = "active" ]; then - validation_pass "$service is active" - else - validation_fail "$service not active (got: $active)" - attribute_issue "$service not active" "archsetup" - return 1 - fi - fi - - return 0 -} - -validate_service_optional() { - local service="$1" - local expected_enabled="$2" - - step "Checking optional service: $service" - - local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled") - if [ "$enabled" = "enabled" ]; then - validation_pass "$service is enabled" - else - validation_warn "$service not enabled (optional)" - fi -} - -validate_zfs_services() { - # Only check if ZFS is installed - if ! ssh_cmd "which zfs" &>> "$LOGFILE"; then - return 0 - fi - - step "Checking ZFS-specific services" - - validate_service_optional "sanoid.timer" "enabled" - - # Check for zfs-scrub timer (pool name varies) - local scrub_enabled - scrub_enabled=$(ssh_cmd "systemctl list-unit-files 'zfs-scrub*' 2>/dev/null | grep -c enabled" | tr -d '[:space:]') - scrub_enabled=${scrub_enabled:-0} - if [ "$scrub_enabled" -gt 0 ]; then - validation_pass "ZFS scrub timer enabled" - else - validation_warn "ZFS scrub timer not found" - fi -} - -validate_btrfs_services() { - # Only check if btrfs root - if ! ssh_cmd "mount | grep 'on / ' | grep -q btrfs"; then - return 0 - fi - - step "Checking btrfs-specific services" - validate_service_optional "grub-btrfsd" "enabled" -} - -validate_service_functions() { - section "Service Functional Tests" - - # UFW functional test - # NOTE: VM environment lacks iptables kernel modules, so UFW cannot activate. - # We only verify it's enabled; active status requires real hardware. - step "Testing UFW functionality" - local ufw_enabled - ufw_enabled=$(ssh_cmd "systemctl is-enabled ufw.service 2>/dev/null || echo disabled") - if [ "$ufw_enabled" = "enabled" ]; then - validation_pass "UFW is enabled (activation requires iptables kernel modules)" - else - validation_fail "UFW not enabled" - attribute_issue "UFW not enabled" "archsetup" - fi - - # fail2ban functional test - step "Testing fail2ban functionality" - if ssh_cmd "fail2ban-client status" &>> "$LOGFILE"; then - validation_pass "fail2ban is responding" - else - validation_fail "fail2ban not responding" - attribute_issue "fail2ban not functioning" "archsetup" - fi - - # DNS resolution test - step "Testing DNS resolution" - if ssh_cmd "resolvectl query archlinux.org" &>> "$LOGFILE"; then - validation_pass "DNS resolution working" - else - validation_warn "DNS resolution test failed (may be network issue)" - fi - - # Docker functional test (if enabled) - if ssh_cmd "systemctl is-enabled docker" &>> "$LOGFILE"; then - step "Testing Docker functionality" - if ssh_cmd "docker info" &>> "$LOGFILE"; then - validation_pass "Docker is responding" - elif ! ssh_cmd "systemctl is-active --quiet docker"; then - # archsetup enables docker for next boot (enable, not enable --now, - # by design — the daemon is heavy). Validation runs pre-reboot, so - # enabled-but-not-started is the correct installed state. - validation_skip "Docker enabled but not started (starts on boot by design)" - else - validation_warn "Docker active but not responding" - fi - fi -} - -#----------------------------------------------------------------------------- -# Developer Tools Validations -#----------------------------------------------------------------------------- - -validate_emacs() { - step "Checking if Emacs is installed" - if ssh_cmd "which emacs" &>> "$LOGFILE"; then - validation_pass "Emacs is installed" - - # Check if config exists - if ssh_cmd "test -d /home/cjennings/.emacs.d"; then - validation_pass "Emacs config directory exists" - - # Check user can access the directory - local result - result=$(ssh_cmd "sudo -u cjennings ls /home/cjennings/.emacs.d > /dev/null 2>&1 && echo OK || echo FAIL") - if [ "$result" = "OK" ]; then - validation_pass "Emacs config readable by user" - else - validation_fail "Emacs config not readable by user (permission issue)" - attribute_issue "Emacs .emacs.d not readable by user" "archsetup" - fi - else - validation_warn "Emacs config directory not found" - fi - else - validation_fail "Emacs not found" - attribute_issue "Emacs not installed" "archsetup" - fi -} - -validate_git_config() { - step "Checking git installation" - if ssh_cmd "which git" &>> "$LOGFILE"; then - validation_pass "git is installed" - else - validation_fail "git not found" - attribute_issue "git not installed" "archsetup" - fi -} - -validate_dev_tools() { - step "Checking developer tools" - local tools="python node npm go rustc" - local missing="" - - for tool in $tools; do - if ! ssh_cmd "which $tool" &>> "$LOGFILE"; then - missing="$missing $tool" - fi - done - - if [ -z "$missing" ]; then - validation_pass "Core dev tools installed" - else - validation_warn "Some dev tools missing:$missing" - fi -} - -#----------------------------------------------------------------------------- -# System Configuration Validations -#----------------------------------------------------------------------------- - -validate_zfs_config() { - step "Checking ZFS configuration (if applicable)" - if ssh_cmd "which zfs" &>> "$LOGFILE"; then - # ZFS is installed, check for sanoid - if ssh_cmd "which sanoid" &>> "$LOGFILE"; then - validation_pass "ZFS with sanoid detected" - else - validation_warn "ZFS detected but sanoid not installed" - fi - else - info "ZFS not installed (non-ZFS system)" - fi -} - -validate_boot_config() { - step "Checking GRUB configuration" - if ssh_cmd "test -f /boot/grub/grub.cfg" &>> "$LOGFILE"; then - validation_pass "GRUB config exists" - else - validation_warn "GRUB config not found (may use different bootloader)" - fi -} - -validate_terminus_font() { - step "Checking terminus-font installation" - if ssh_cmd "pacman -Q terminus-font" &>> "$LOGFILE"; then - validation_pass "terminus-font package installed" - else - validation_fail "terminus-font package not installed" - attribute_issue "terminus-font not installed via pacman" "archsetup" - fi -} - -validate_mkinitcpio_hooks() { - step "Checking mkinitcpio HOOKS configuration" - local hooks=$(ssh_cmd "grep '^HOOKS=' /etc/mkinitcpio.conf") - local is_zfs=$(ssh_cmd "findmnt -n -o FSTYPE / 2>/dev/null") - - if [ "$is_zfs" = "zfs" ]; then - # ZFS system: must use udev, not systemd - if echo "$hooks" | grep -q '\budev\b'; then - validation_pass "ZFS system uses udev hook (correct)" - elif echo "$hooks" | grep -q '\bsystemd\b'; then - validation_fail "ZFS system uses systemd hook (will break boot)" - attribute_issue "mkinitcpio uses systemd hook on ZFS system" "archsetup" - else - validation_warn "Could not determine init hook type" - fi - else - # Non-ZFS: systemd hook is fine - if echo "$hooks" | grep -q '\bsystemd\b'; then - validation_pass "Non-ZFS system uses systemd hook" - elif echo "$hooks" | grep -q '\budev\b'; then - validation_pass "Non-ZFS system uses udev hook" - fi - fi -} - -validate_initramfs_consolefont() { - step "Checking console font in initramfs" - local font_in_initramfs=$(ssh_cmd "lsinitcpio /boot/initramfs-linux*.img 2>/dev/null | grep -c 'consolefont.psf\\|ter-'") - - if [ "${font_in_initramfs:-0}" -gt 0 ]; then - validation_pass "Console font included in initramfs" - else - validation_warn "Console font may not be in initramfs" - fi -} - -validate_nvme_module() { - step "Checking NVMe module configuration" - local has_nvme=$(ssh_cmd "ls /dev/nvme* 2>/dev/null | head -1") - - if [ -n "$has_nvme" ]; then - # System has NVMe drives - local modules=$(ssh_cmd "grep '^MODULES=' /etc/mkinitcpio.conf") - if echo "$modules" | grep -q 'nvme'; then - validation_pass "NVMe module in mkinitcpio MODULES" - else - validation_warn "NVMe system but nvme not in MODULES (may cause slow boot)" - fi - else - info "No NVMe drives detected" - fi -} - -validate_autologin_config() { - step "Checking autologin configuration" - if ssh_cmd "test -f /etc/systemd/system/getty@tty1.service.d/autologin.conf" &>> "$LOGFILE"; then - validation_pass "Autologin configured" - else - info "Autologin not configured (may be intentional)" - fi -} - -validate_gnome_keyring_setup() { - step "Checking gnome-keyring pre-configuration" - local keyring_dir="/home/cjennings/.local/share/keyrings" - - # Check directory exists - if ! ssh_cmd "test -d $keyring_dir"; then - validation_fail "Keyring directory not created" - attribute_issue "gnome-keyring directory not pre-created" "archsetup" - return 1 - fi - - # Check directory permissions (should be 700) - local perms=$(ssh_cmd "stat -c '%a' $keyring_dir") - if [ "$perms" != "700" ]; then - validation_fail "Keyring directory has wrong permissions: $perms (expected 700)" - attribute_issue "gnome-keyring directory wrong permissions" "archsetup" - return 1 - fi - - # Check ownership - local owner=$(ssh_cmd "stat -c '%U' $keyring_dir") - if [ "$owner" != "cjennings" ]; then - validation_fail "Keyring directory owned by $owner (expected cjennings)" - attribute_issue "gnome-keyring directory wrong ownership" "archsetup" - return 1 - fi - - # Check default file exists and contains "login" - local default_keyring=$(ssh_cmd "cat $keyring_dir/default 2>/dev/null") - if [ "$default_keyring" != "login" ]; then - validation_fail "Default keyring not set to 'login' (got: '$default_keyring')" - attribute_issue "gnome-keyring default not set to login" "archsetup" - return 1 - fi - - validation_pass "gnome-keyring pre-configured (default=login, dir=700)" -} - -#----------------------------------------------------------------------------- -# Archsetup-Specific Validations -#----------------------------------------------------------------------------- - -validate_archsetup_log() { - step "Checking archsetup log for errors" - local error_count - # Use grep -h to suppress filenames, then wc -l to count total matches - error_count=$(ssh_cmd "grep -h '^Error:' /var/log/archsetup-*.log 2>/dev/null | wc -l" | tr -d '[:space:]') - error_count=${error_count:-0} - - if [ "$error_count" = "0" ]; then - validation_pass "No errors in archsetup log" - else - validation_fail "Found $error_count errors in archsetup log" - attribute_issue "Errors in archsetup log: $error_count" "archsetup" - fi -} - -validate_state_markers() { - step "Checking archsetup state markers" - local state_count=$(ssh_cmd "ls /var/lib/archsetup/state/ 2>/dev/null | wc -l") - - if [ "$state_count" -ge 12 ]; then - validation_pass "All 12 installation steps completed" - else - validation_warn "Only $state_count/12 steps completed" - fi -} - -#============================================================================= # ISSUE REPORTING #============================================================================= @@ -1139,18 +312,3 @@ EOF info "Issue report saved: $report_file" } -#============================================================================= -# MAIN VALIDATION ENTRY POINT -#============================================================================= - -run_full_validation() { - local output_dir="$1" - local archzfs_inbox="${2:-}" - - run_all_validations - analyze_log_diff "$output_dir" - generate_issue_report "$output_dir" "$archzfs_inbox" - - # Return success if no failures - [ $VALIDATION_FAILED -eq 0 ] -} diff --git a/scripts/testing/lib/vm-utils.sh b/scripts/testing/lib/vm-utils.sh index d029d58..b85e773 100755 --- a/scripts/testing/lib/vm-utils.sh +++ b/scripts/testing/lib/vm-utils.sh @@ -11,13 +11,26 @@ # VM configuration defaults VM_CPUS="${VM_CPUS:-4}" -VM_RAM="${VM_RAM:-4096}" # MB +# 8 GiB headroom for AUR builds: makepkg runs -j$VM_CPUS, and parallel cc1plus +# (~700 MB each on heavy C++ packages) OOM-killed under the old 4 GiB default. +VM_RAM="${VM_RAM:-8192}" # MB VM_DISK_SIZE="${VM_DISK_SIZE:-50}" # GB +# Filesystem profile: selects which base image + archangel config the harness +# targets. "btrfs" is the historical default (its image name stays unsuffixed +# so existing base images keep working); "zfs" gets its own image, since the +# two on-disk layouts can't share a disk. Honoured by init_vm_paths below. +FS_PROFILE="${FS_PROFILE:-btrfs}" + # SSH configuration SSH_PORT="${SSH_PORT:-2222}" SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10" ROOT_PASSWORD="${ROOT_PASSWORD:-archsetup}" +# Set by inject_root_key once a root key is authorized in the VM. When set, the +# ssh/scp helpers add "-i <key>" so they keep working after archsetup hardens +# sshd to PermitRootLogin prohibit-password (which kills root *password* login +# but still allows key auth). Left unquoted at use sites, like SSH_OPTS. +SSH_KEY_OPT="${SSH_KEY_OPT:-}" # OVMF firmware paths OVMF_CODE="/usr/share/edk2/x64/OVMF_CODE.4m.fd" @@ -37,9 +50,22 @@ init_vm_paths() { local images_dir="${1:-$VM_IMAGES_DIR}" [ -z "$images_dir" ] && fatal "VM_IMAGES_DIR not set" + case "$FS_PROFILE" in + btrfs|zfs) ;; + *) fatal "Invalid FS_PROFILE: $FS_PROFILE (must be 'btrfs' or 'zfs')" ;; + esac + VM_IMAGES_DIR="$images_dir" - DISK_PATH="$VM_IMAGES_DIR/archsetup-base.qcow2" - OVMF_VARS="$VM_IMAGES_DIR/OVMF_VARS.fd" + # btrfs keeps the legacy unsuffixed name; other profiles get a suffix so + # their images sit side by side without clobbering each other. + local img_suffix="" + [ "$FS_PROFILE" != "btrfs" ] && img_suffix="-$FS_PROFILE" + DISK_PATH="$VM_IMAGES_DIR/archsetup-base${img_suffix}.qcow2" + # Per-profile NVRAM: UEFI boot entries live here, outside the qcow2, so a + # disk-snapshot revert can't restore them. Sharing one file across profiles + # let a zfs run's ZFSBootMenu entries clobber the btrfs GRUB entry, leaving + # the btrfs base unbootable (no removable ESP fallback to recover from). + OVMF_VARS="$VM_IMAGES_DIR/OVMF_VARS${img_suffix}.fd" PID_FILE="$VM_IMAGES_DIR/qemu.pid" MONITOR_SOCK="$VM_IMAGES_DIR/qemu-monitor.sock" SERIAL_LOG="$VM_IMAGES_DIR/qemu-serial.log" @@ -351,7 +377,7 @@ wait_for_ssh() { progress "Waiting for SSH on localhost:$SSH_PORT..." while [ "$elapsed" -lt "$timeout" ]; do - if sshpass -p "$password" ssh $SSH_OPTS -p "$SSH_PORT" root@localhost true 2>/dev/null; then + if sshpass -p "$password" ssh $SSH_OPTS $SSH_KEY_OPT -p "$SSH_PORT" root@localhost true 2>/dev/null; then success "SSH is available" return 0 fi @@ -367,7 +393,7 @@ wait_for_ssh() { vm_exec() { local password="${1:-$ROOT_PASSWORD}" shift - sshpass -p "$password" ssh $SSH_OPTS \ + sshpass -p "$password" ssh $SSH_OPTS $SSH_KEY_OPT \ -o ServerAliveInterval=30 -o ServerAliveCountMax=10 \ -p "$SSH_PORT" root@localhost "$@" 2>> "$LOGFILE" } @@ -379,7 +405,7 @@ copy_to_vm() { local password="${3:-$ROOT_PASSWORD}" step "Copying $(basename "$local_file") to VM:$remote_path" - if sshpass -p "$password" scp $SSH_OPTS -P "$SSH_PORT" \ + if sshpass -p "$password" scp $SSH_OPTS $SSH_KEY_OPT -P "$SSH_PORT" \ "$local_file" "root@localhost:$remote_path" >> "$LOGFILE" 2>&1; then success "File copied to VM" return 0 @@ -396,7 +422,7 @@ copy_from_vm() { local password="${3:-$ROOT_PASSWORD}" step "Copying $remote_file from VM" - if sshpass -p "$password" scp $SSH_OPTS -P "$SSH_PORT" \ + if sshpass -p "$password" scp $SSH_OPTS $SSH_KEY_OPT -P "$SSH_PORT" \ "root@localhost:$remote_file" "$local_path" >> "$LOGFILE" 2>&1; then success "File copied from VM" return 0 @@ -405,3 +431,31 @@ copy_from_vm() { return 1 fi } + +# inject_root_key <key_path> +# Authorize a throwaway root key over the initial password session and switch +# all the helpers above to key auth (sets SSH_KEY_OPT + ROOT_SSH_KEY). Call once, +# right after wait_for_ssh and before running archsetup: archsetup sets +# PermitRootLogin prohibit-password and reloads sshd partway through, which kills +# root *password* login. Without a key in place first, every SSH after that step +# fails and the run aborts before any validation. Key auth survives the hardening. +# Targets root@$VM_IP on $SSH_PORT so it works for both the local VM runner +# (VM_IP=localhost, port 2222) and the bare-metal runner (VM_IP=host, port 22). +inject_root_key() { + local key="$1" + rm -f "$key" "$key.pub" + if ! ssh-keygen -t ed25519 -N "" -q -f "$key"; then + warn "Root key generation failed - run may break at sshd hardening" + return 1 + fi + if sshpass -p "$ROOT_PASSWORD" ssh $SSH_OPTS -p "$SSH_PORT" "root@${VM_IP:-localhost}" \ + "mkdir -p /root/.ssh && chmod 700 /root/.ssh && cat >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys" \ + < "$key.pub" >> "$LOGFILE" 2>&1; then + SSH_KEY_OPT="-i $key" + export ROOT_SSH_KEY="$key" + success "Root SSH key authorized (survives sshd prohibit-password hardening)" + return 0 + fi + warn "Root key authorization failed - run may break at sshd hardening" + return 1 +} diff --git a/scripts/testing/maint-scenarios/10-journal-vacuum.sh b/scripts/testing/maint-scenarios/10-journal-vacuum.sh new file mode 100644 index 0000000..b61de49 --- /dev/null +++ b/scripts/testing/maint-scenarios/10-journal-vacuum.sh @@ -0,0 +1,30 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: the journal vacuum executes against live journald at the TOML floor. +# +# What this proves: the TOML edit reaches the remedy's argv (vacuum target +# and warn threshold are the same key), journalctl --vacuum-size runs for +# real against live journald, and the metric grades ok against the edited +# floor afterwards. +# +# What it deliberately does NOT prove: size reduction. The key is int-GB and +# bottoms out at 1, vacuum only touches archived files, and a VM-sized +# journal never nears 1 GB — a genuine reduction test would need journald +# rate-limit surgery plus ~1 GB of spam per run. A pre/post usage comparison +# is also unsound here: journald keeps landing in-flight writes (the spam is +# async, and every ssh hop logs), so usage can grow across a correct vacuum. +SCENARIO_DESC="journal vacuum runs at the TOML size floor" +SCENARIO_GROUP="logs" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "sed -i 's/^journal_disk_warn_gb *=.*/journal_disk_warn_gb = 1/' /root/.config/archsetup/maintenance-thresholds.toml" \ + && mexec "for i in \$(seq 1 20); do head -c 900000 /dev/urandom | base64 | systemd-cat -t maint-scenario-spam; done; journalctl --rotate; journalctl --flush >/dev/null 2>&1 || true" +} + +scenario_fix() { + mfix journal_vacuum +} + +scenario_assert() { + massert_metric journal_disk ok +} diff --git a/scripts/testing/maint-scenarios/11-coredump-age-out.sh b/scripts/testing/maint-scenarios/11-coredump-age-out.sh new file mode 100644 index 0000000..06d6fa0 --- /dev/null +++ b/scripts/testing/maint-scenarios/11-coredump-age-out.sh @@ -0,0 +1,21 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: coredump files beyond the forensic window are aged out. +# +# coredumpctl has no clean verb, so the remedy is a find -mtime +N -delete +# over /var/lib/systemd/coredump. Break plants one file older than the +# window and one fresh file; assert checks exactly the old one is gone. +SCENARIO_DESC="aged coredump files removed, fresh ones kept" +SCENARIO_GROUP="logs" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "mkdir -p /var/lib/systemd/coredump && touch -d '30 days ago' '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && touch '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'" +} + +scenario_fix() { + mfix coredump_clean +} + +scenario_assert() { + mexec "test ! -e '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && test -e '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'" +} diff --git a/scripts/testing/maint-scenarios/20-cache-keep3.sh b/scripts/testing/maint-scenarios/20-cache-keep3.sh new file mode 100644 index 0000000..a9a6ee5 --- /dev/null +++ b/scripts/testing/maint-scenarios/20-cache-keep3.sh @@ -0,0 +1,24 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: paccache -rk3 trims an installed package's cache to 3 versions. +# +# paccache decides from filenames, so five fake versions of a package that +# IS installed (bash) stand in for a fat cache. The bootstrap may leave a +# real bash package in the cache too, so the assert uses the ordering +# invariant rather than a count: the two newest fakes survive any keep-3 +# window that contains them, and the two oldest fall outside it whether or +# not a real (newer) version is cached alongside. +SCENARIO_DESC="pacman cache trimmed to three versions per package" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "for v in 1.0.1 1.0.2 1.0.3 1.0.4 1.0.5; do touch /var/cache/pacman/pkg/bash-\$v-1-x86_64.pkg.tar.zst; done; ls /var/cache/pacman/pkg/ | grep -c '^bash-'" +} + +scenario_fix() { + mfix cache_clean +} + +scenario_assert() { + mexec "test -e /var/cache/pacman/pkg/bash-1.0.5-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-1.0.4-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.2-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.1-1-x86_64.pkg.tar.zst" +} diff --git a/scripts/testing/maint-scenarios/21-cache-uninstalled.sh b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh new file mode 100644 index 0000000..ea788e8 --- /dev/null +++ b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: paccache -ruk0 drops every cached version of uninstalled packages. +# +# Fake versions of a package that is NOT installed disappear entirely, while +# an installed package's cached version survives the uninstalled-only pass. +SCENARIO_DESC="cached versions of uninstalled packages dropped" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "touch /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst" +} + +scenario_fix() { + mfix cache_clean_uninstalled +} + +scenario_assert() { + mexec "test ! -e /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst" +} diff --git a/scripts/testing/maint-scenarios/22-orphan-remove.sh b/scripts/testing/maint-scenarios/22-orphan-remove.sh new file mode 100644 index 0000000..8412128 --- /dev/null +++ b/scripts/testing/maint-scenarios/22-orphan-remove.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a real orphan is created, detected, and removed by name. +# +# Installing `tree` --asdeps with no dependents makes it a true orphan +# (pacman -Qtdq lists it). The remedy is item-based: maint fix orphan_remove +# tree. Assert covers both the package state and the metric verdict. +SCENARIO_DESC="orphan package removed via the item-based remedy" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "pacman -S --noconfirm --asdeps tree && pacman -Qtdq | grep -qx tree" +} + +scenario_fix() { + mfix orphan_remove tree +} + +scenario_assert() { + mexec "! pacman -Qi tree >/dev/null 2>&1 && ! pacman -Qtdq | grep -qx tree" \ + && massert_metric pkg_orphans ok +} diff --git a/scripts/testing/maint-scenarios/23-pacnew-delete.sh b/scripts/testing/maint-scenarios/23-pacnew-delete.sh new file mode 100644 index 0000000..23cd8d1 --- /dev/null +++ b/scripts/testing/maint-scenarios/23-pacnew-delete.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a .pacnew file is deleted through the validated priv verb. +# +# pacdiff --output finds the planted file; the remedy takes the path as its +# item and rm's it. A neighbouring non-pacnew file proves the verb's path +# validation keeps the blast radius to the named file. +SCENARIO_DESC="planted .pacnew removed by path" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "cp /etc/pacman.conf /etc/pacman.conf.pacnew && pacdiff --output | grep -q pacman.conf.pacnew" +} + +scenario_fix() { + mfix pacnew_delete /etc/pacman.conf.pacnew +} + +scenario_assert() { + mexec "test ! -e /etc/pacman.conf.pacnew && test -e /etc/pacman.conf" \ + && massert_metric pkg_pacnew ok +} diff --git a/scripts/testing/maint-scenarios/30-unit-reset.sh b/scripts/testing/maint-scenarios/30-unit-reset.sh new file mode 100644 index 0000000..07eed53 --- /dev/null +++ b/scripts/testing/maint-scenarios/30-unit-reset.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a failed unit's state is reset. +# +# A transient oneshot that exits 1 lands in systemctl --failed; the remedy +# resets it and the failed list returns to its pre-break size. The unit name +# rides the remedy's item argument, exercising the priv verb's unit-name +# validation. The base image carries its own failed unit (grub-btrfsd — no +# snapshot dirs in the VM), so the assert scopes to this scenario's unit and +# the count delta, never the absolute failed list. +SCENARIO_DESC="failed unit reset clears the failed list" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="any" + +scenario_break() { + PRE_FAILED_COUNT=$(mexec "systemctl --failed --no-legend | wc -l") \ + && mexec "systemd-run --unit=maint-scenario-fail --service-type=oneshot --no-block /bin/false; sleep 2; systemctl is-failed maint-scenario-fail.service" +} + +scenario_fix() { + mfix unit_reset maint-scenario-fail.service +} + +scenario_assert() { + local pre="$PRE_FAILED_COUNT" + unset PRE_FAILED_COUNT # don't leak scenario state past this scenario + mexec "! systemctl is-failed maint-scenario-fail.service >/dev/null 2>&1" \ + && [ "$(mexec "systemctl --failed --no-legend | wc -l")" = "$pre" ] +} diff --git a/scripts/testing/maint-scenarios/31-fstrim-enable.sh b/scripts/testing/maint-scenarios/31-fstrim-enable.sh new file mode 100644 index 0000000..2d548a1 --- /dev/null +++ b/scripts/testing/maint-scenarios/31-fstrim-enable.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: fstrim.timer enabled and started from the disabled default. +# +# The base image ships util-linux with fstrim.timer disabled — the exact +# state the metric flags. The remedy is systemctl enable --now. +SCENARIO_DESC="fstrim.timer enabled and started" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="btrfs" + +scenario_break() { + # Default state, but pin it so the scenario stays honest if a future + # base image enables the timer. + mexec "systemctl disable --now fstrim.timer >/dev/null 2>&1 || true; ! systemctl is-enabled fstrim.timer >/dev/null 2>&1" +} + +scenario_fix() { + mfix fstrim_enable +} + +scenario_assert() { + mexec "systemctl is-enabled fstrim.timer && systemctl is-active fstrim.timer" +} diff --git a/scripts/testing/maint-scenarios/32-cron-enable.sh b/scripts/testing/maint-scenarios/32-cron-enable.sh new file mode 100644 index 0000000..39d4de6 --- /dev/null +++ b/scripts/testing/maint-scenarios/32-cron-enable.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: cronie enabled and started (the stopped-cron posture). +# +# Bootstrap installs cronie but never enables it — the broken state is the +# package's default. The remedy enables + starts the service. +SCENARIO_DESC="cronie enabled and started from the stopped state" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "systemctl disable --now cronie.service >/dev/null 2>&1 || true; ! systemctl is-active cronie.service >/dev/null 2>&1" +} + +scenario_fix() { + mfix cron_enable +} + +scenario_assert() { + mexec "systemctl is-enabled cronie.service && systemctl is-active cronie.service" +} diff --git a/scripts/testing/run-maint-nspawn.sh b/scripts/testing/run-maint-nspawn.sh new file mode 100644 index 0000000..04d4a32 --- /dev/null +++ b/scripts/testing/run-maint-nspawn.sh @@ -0,0 +1,268 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# Run pacman-level maint remedy scenarios in a systemd-nspawn container +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# The fast lane for the maint scenario suite: pure pacman-level cases (the +# "packages" scenario group — cache trims, orphan removal, pacnew deletion) +# don't need a booted machine, so they run in seconds against a pacstrap'd +# rootfs instead of minutes against the VM. Everything else (systemd, logs) +# needs pid 1 and journald — that stays in run-maint-scenarios.sh. +# +# The scenario files are shared with the VM runner and see the same helpers +# (mexec/mmaint/mfix/massert_metric); only the transport differs — the +# helper trio is deliberately duplicated between the two runners rather +# than abstracted over ssh-vs-nspawn. +# +# The rootfs is cached at $NSPAWN_ROOT and rebuilt with --fresh. Building +# and running need root (pacstrap, systemd-nspawn) via sudo. +# +# Usage: run-maint-nspawn.sh [--list] [--fresh] +# --list print the validated scenario plan and exit (no root needed) +# --fresh delete the cached rootfs and pacstrap a new one +# Env: MAINT_SCENARIO_DIR scenario dir (default: maint-scenarios/) +# MAINT_SRC maint package source tree +# (default: ~/.dotfiles/maint/src/maint) +# NSPAWN_ROOT rootfs dir (default: /var/tmp/archsetup-maint-nspawn) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" + +SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" +MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" +THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml" +NSPAWN_ROOT="${NSPAWN_ROOT:-/var/tmp/archsetup-maint-nspawn}" +NSPAWN_GROUP="packages" + +LIST_ONLY=false +FRESH=false + +usage() { + echo "Usage: $0 [--list] [--fresh]" + echo " --list print the validated scenario plan and exit" + echo " --fresh rebuild the cached rootfs from scratch" + echo "Env: MAINT_SCENARIO_DIR, MAINT_SRC, NSPAWN_ROOT" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --list) LIST_ONLY=true; shift ;; + --fresh) FRESH=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# ─── Plan: the pacman-level subset ─────────────────────────────────── +# Same contract validation as the VM runner; the selection rule is the +# group name — "packages" is pacman-level by construction. + +S_FILES=() S_NAMES=() S_DESCS=() + +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +build_plan() { + [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" + local f base + for f in "$SCENARIO_DIR"/*.sh; do + [ -e "$f" ] || break + _validate_scenario "$f" + [ "$(_scenario_var "$f" SCENARIO_GROUP)" = "$NSPAWN_GROUP" ] || continue + base=$(basename "$f" .sh) + S_FILES+=("$f") + S_NAMES+=("${base#[0-9][0-9]-}") + S_DESCS+=("$(_scenario_var "$f" SCENARIO_DESC)") + done + [ ${#S_FILES[@]} -gt 0 ] \ + || fatal "no '$NSPAWN_GROUP'-group scenarios in $SCENARIO_DIR" +} + +print_plan() { + local i + echo "maint nspawn scenario plan (group: $NSPAWN_GROUP)" + for i in "${!S_FILES[@]}"; do + echo " ${S_NAMES[$i]} — ${S_DESCS[$i]}" + done +} + +if [ "$LIST_ONLY" = "true" ]; then + LOGFILE=/dev/null + build_plan + print_plan + exit 0 +fi + +# ─── Run ───────────────────────────────────────────────────────────── + +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +RESULTS_DIR="$PROJECT_ROOT/test-results/maint-nspawn-$TIMESTAMP" +mkdir -p "$RESULTS_DIR" +LOGFILE="$RESULTS_DIR/scenarios.log" +init_logging "$LOGFILE" + +build_plan + +section "Maint Remedy Scenarios (nspawn): $TIMESTAMP" +info "Rootfs: $NSPAWN_ROOT" +info "Maint source: $MAINT_SRC" +print_plan | tee -a "$LOGFILE" + +command -v pacstrap >/dev/null \ + || fatal "pacstrap not found — pacman -S arch-install-scripts" +command -v systemd-nspawn >/dev/null || fatal "systemd-nspawn not found" +[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC" +[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML" +sudo -n true 2>/dev/null || fatal "needs passwordless sudo (pacstrap/nspawn)" + +# ─── Target helpers (available to scenario scripts) ────────────────── + +mexec() { # run a command in the container as root + sudo systemd-nspawn -q -D "$NSPAWN_ROOT" --pipe \ + /bin/bash -c "$*" 2>> "$LOGFILE" +} + +mmaint() { + mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*" +} + +mfix() { + local id="$1"; shift || true + step "maint fix $id ${*:+$* }--dry-run" + mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \ + || { error "dry-run failed: maint fix $id $*"; return 1; } + step "maint fix $id $*" + mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE" + return "${PIPESTATUS[0]}" +} + +massert_metric() { # <metric-id> <expected-severity> + local mid="$1" want="$2" got + got=$(mmaint status --json | python3 -c " +import json, sys +env = json.load(sys.stdin) +ms = [m for m in env['metrics'] if m['id'] == '$mid'] +print(ms[0]['severity'] if ms else 'MISSING') +") + if [ "$got" = "$want" ]; then + success "metric $mid is $want" + return 0 + fi + error "metric $mid: expected $want, got $got" + return 1 +} + +# ─── Rootfs ────────────────────────────────────────────────────────── + +if [ "$FRESH" = "true" ] && [ -d "$NSPAWN_ROOT" ]; then + step "Removing cached rootfs (--fresh)" + sudo rm -rf "$NSPAWN_ROOT" +fi + +if [ ! -x "$NSPAWN_ROOT/usr/bin/python3" ]; then + section "Building rootfs (pacstrap)" + sudo mkdir -p "$NSPAWN_ROOT" + # shellcheck disable=SC2024 # the log is user-owned by design + sudo pacstrap -c -K "$NSPAWN_ROOT" base python pacman-contrib expac \ + >> "$LOGFILE" 2>&1 || fatal "pacstrap failed (see $LOGFILE)" + success "Rootfs built" +else + info "Reusing cached rootfs (--fresh to rebuild)" +fi + +section "Installing maint tree + thresholds into the container" +sudo rm -rf "$NSPAWN_ROOT/root/maint-pkg" +sudo mkdir -p "$NSPAWN_ROOT/root/maint-pkg" "$NSPAWN_ROOT/root/.config/archsetup" +sudo cp -r "$MAINT_SRC" "$NSPAWN_ROOT/root/maint-pkg/" +sudo find "$NSPAWN_ROOT/root/maint-pkg" -name '__pycache__' -type d \ + -exec rm -rf {} + 2>/dev/null || true +sudo cp "$THRESHOLDS_TOML" \ + "$NSPAWN_ROOT/root/.config/archsetup/maintenance-thresholds.toml" + +step "Smoke: maint status --json runs in the container" +mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \ + || fatal "maint status failed in the container" +success "Container ready" + +# ─── Execution ─────────────────────────────────────────────────────── + +PASS=() FAIL=() + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} + +for i in "${!S_FILES[@]}"; do + run_scenario "$i" +done + +# ─── Report ────────────────────────────────────────────────────────── + +section "Results" +for n in "${PASS[@]}"; do success "PASS $n"; done +for n in "${FAIL[@]}"; do error "FAIL $n"; done +info "Log: $LOGFILE" +info "Rootfs kept for reuse: $NSPAWN_ROOT (--fresh to rebuild)" + +if [ ${#FAIL[@]} -gt 0 ]; then + error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed" + exit 1 +fi +success "All ${#PASS[@]} scenarios passed" +exit 0 diff --git a/scripts/testing/run-maint-scenarios.sh b/scripts/testing/run-maint-scenarios.sh new file mode 100644 index 0000000..448f6d9 --- /dev/null +++ b/scripts/testing/run-maint-scenarios.sh @@ -0,0 +1,374 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# Run maintenance-console remedy scenarios in the test VM +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# Layer 3 of the maint test strategy: each scenario breaks the VM in a known +# way over ssh, runs the real `maint fix <id>` inside the VM, and asserts the +# post-state. Scenarios live in maint-scenarios/*.sh and declare a GROUP; +# scenarios in one group are non-conflicting and share a VM boot, and a +# stop -> restore -> boot cycle runs only between groups (snapshot ops need +# the VM stopped). A scenario that fails may leave residue behind for its +# in-group successors -- the group restore boundary contains it, so read a +# multi-failure group front to back. +# +# Boot plan: +# 1. restore clean-install, boot, bootstrap (packages + maint tree + TOML) +# 2. stop, snapshot maint-ready +# 3. per group: boot, run scenarios, stop, restore maint-ready +# 4. restore clean-install, delete maint-ready (base stays pristine) +# +# Usage: run-maint-scenarios.sh [--list] [--group NAME] [--keep] +# --list print the validated batch plan and exit (no VM, no KVM needed) +# --group run (or list) only the named group +# --keep keep the VM running in its post-run state; skip the restores +# Env: FS_PROFILE=btrfs|zfs base image + scenario profile filter +# MAINT_SCENARIO_DIR scenario dir (default: maint-scenarios/) +# MAINT_SRC maint package source tree +# (default: ~/.dotfiles/maint/src/maint) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" + +SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" +MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" +THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml" +ROOT_PASSWORD="archsetup" +READY_SNAPSHOT="maint-ready" + +LIST_ONLY=false +ONLY_GROUP="" +KEEP_VM=false + +usage() { + echo "Usage: $0 [--list] [--group NAME] [--keep]" + echo " --list print the validated scenario plan and exit (no VM)" + echo " --group run only the named scenario group" + echo " --keep keep the VM running after the run (skip restores)" + echo "Env: FS_PROFILE=btrfs|zfs, MAINT_SCENARIO_DIR, MAINT_SRC" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --list) LIST_ONLY=true; shift ;; + --group) ONLY_GROUP="${2:?--group requires a value}"; shift 2 ;; + --keep) KEEP_VM=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# ─── Plan: enumerate, validate, group ──────────────────────────────── +# The plan layer is pure -- no VM, no KVM -- so --list works anywhere and +# the unit suite (tests/maint-scenarios/) exercises it directly. + +# Parallel arrays over scenarios, in file order. +S_FILES=() S_NAMES=() S_DESCS=() S_GROUPS=() +# Groups in first-appearance order. +GROUP_ORDER=() + +# Extract one declared var from a scenario file by sourcing it in a clean +# subshell. Scenario files must only define vars + functions; any top-level +# command fails here (no helpers exist), which is the contract. +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +_profile_matches() { # <profiles> -> 0 if scenario runs under $FS_PROFILE + local p + for p in $1; do + [ "$p" = "any" ] || [ "$p" = "$FS_PROFILE" ] && return 0 + done + return 1 +} + +build_plan() { + [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" + local f base name desc group profiles seen_groups=" " + local any=false + for f in "$SCENARIO_DIR"/*.sh; do + [ -e "$f" ] || break + any=true + _validate_scenario "$f" + base=$(basename "$f" .sh) + name="${base#[0-9][0-9]-}" + desc=$(_scenario_var "$f" SCENARIO_DESC) + group=$(_scenario_var "$f" SCENARIO_GROUP) + profiles=$(_scenario_var "$f" SCENARIO_PROFILES) + _profile_matches "$profiles" || continue + [ -n "$ONLY_GROUP" ] && [ "$group" != "$ONLY_GROUP" ] && continue + S_FILES+=("$f"); S_NAMES+=("$name") + S_DESCS+=("$desc"); S_GROUPS+=("$group") + case "$seen_groups" in + *" $group "*) ;; + *) GROUP_ORDER+=("$group"); seen_groups="$seen_groups$group " ;; + esac + done + [ "$any" = "true" ] || fatal "no scenario files in $SCENARIO_DIR" + if [ -n "$ONLY_GROUP" ] && [ ${#S_FILES[@]} -eq 0 ]; then + fatal "no scenarios match group '$ONLY_GROUP' (profile $FS_PROFILE)" + fi + [ ${#S_FILES[@]} -gt 0 ] \ + || fatal "no scenarios match profile $FS_PROFILE" +} + +print_plan() { + local g i + echo "maint scenario plan (profile: $FS_PROFILE)" + for g in "${GROUP_ORDER[@]}"; do + echo "group $g:" + for i in "${!S_FILES[@]}"; do + [ "${S_GROUPS[$i]}" = "$g" ] \ + && echo " ${S_NAMES[$i]} — ${S_DESCS[$i]}" + done + done +} + +# --list needs no logging file, VM, or KVM. Give fatal() a LOGFILE target +# only when we actually run. +if [ "$LIST_ONLY" = "true" ]; then + LOGFILE=/dev/null + build_plan + print_plan + exit 0 +fi + +# ─── Run ───────────────────────────────────────────────────────────── + +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +RESULTS_DIR="$PROJECT_ROOT/test-results/maint-$TIMESTAMP" +mkdir -p "$RESULTS_DIR" +LOGFILE="$RESULTS_DIR/scenarios.log" +init_logging "$LOGFILE" +init_vm_paths "$PROJECT_ROOT/vm-images" + +build_plan + +section "Maint Remedy Scenarios: $TIMESTAMP" +info "Profile: $FS_PROFILE (image: $(basename "$DISK_PATH"))" +info "Maint source: $MAINT_SRC" +print_plan | tee -a "$LOGFILE" + +[ -f "$DISK_PATH" ] || fatal "base disk not found: $DISK_PATH — build it: FS_PROFILE=$FS_PROFILE $SCRIPT_DIR/create-base-vm.sh" +snapshot_exists "$DISK_PATH" "clean-install" \ + || fatal "snapshot 'clean-install' not found on $DISK_PATH" +[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC" +[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML" +check_prerequisites || fatal "missing prerequisites" + +CLEANUP_DONE=0 +cleanup_scenarios() { + [ "$CLEANUP_DONE" = "1" ] && return 0 + CLEANUP_DONE=1 + [ "$KEEP_VM" = "true" ] && return 0 + # Never silent: a failed restore/delete here leaves the base image dirty + # for the next `make test`, so name it even though the trap can't abort. + stop_qemu 2>/dev/null \ + || echo "[!] cleanup: stop_qemu failed — VM may still hold the image" >&2 + restore_snapshot "$DISK_PATH" "clean-install" 2>/dev/null \ + || echo "[!] cleanup: clean-install restore FAILED — base image is dirty" >&2 + if snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT"; then + delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" 2>/dev/null \ + || echo "[!] cleanup: stray '$READY_SNAPSHOT' snapshot left on the base image" >&2 + fi +} +trap cleanup_scenarios EXIT + +# ─── Target helpers (available to scenario scripts) ────────────────── + +mexec() { # run a command in the VM as root + vm_exec "$ROOT_PASSWORD" "$@" +} + +# maint invocation inside the VM. Root runs the verbs directly, so +# MAINT_SUDO is empty (same switch the unit suites use). +mmaint() { + mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*" +} + +# Run one remedy: dry-run first (argv parity gate — the printed argv is the +# arm-press string), then for real. Fails if either exits non-zero. +mfix() { + local id="$1"; shift || true + step "maint fix $id ${*:+$* }--dry-run" + mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \ + || { error "dry-run failed: maint fix $id $*"; return 1; } + step "maint fix $id $*" + mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE" + return "${PIPESTATUS[0]}" +} + +# Assert a metric's severity in `maint status --json`. +massert_metric() { # <metric-id> <expected-severity> + local mid="$1" want="$2" got + got=$(mmaint status --json | python3 -c " +import json, sys +env = json.load(sys.stdin) +ms = [m for m in env['metrics'] if m['id'] == '$mid'] +print(ms[0]['severity'] if ms else 'MISSING') +") + if [ "$got" = "$want" ]; then + success "metric $mid is $want" + return 0 + fi + error "metric $mid: expected $want, got $got" + return 1 +} + +# ─── Bootstrap ─────────────────────────────────────────────────────── + +boot_vm() { + start_qemu "$DISK_PATH" "disk" "" "none" || fatal "failed to start VM" + wait_for_ssh "$ROOT_PASSWORD" 180 || fatal "VM SSH not available" +} + +bootstrap_vm() { + section "Bootstrap: packages + maint tree + thresholds" + step "Refreshing keyring + installing runtime deps" + # The base image can be months old: refresh the keyring first so the + # dep install doesn't die on expired signatures. + mexec "pacman -Sy --noconfirm archlinux-keyring" >> "$LOGFILE" 2>&1 \ + || fatal "keyring refresh failed (network up?)" + mexec "pacman -S --noconfirm python pacman-contrib expac cronie" \ + >> "$LOGFILE" 2>&1 || fatal "dep install failed" + + step "Copying maint package tree" + local tarball + tarball=$(mktemp) + tar -C "$(dirname "$MAINT_SRC")" --exclude='__pycache__' -czf "$tarball" \ + "$(basename "$MAINT_SRC")" + copy_to_vm "$tarball" "/tmp/maint-src.tgz" "$ROOT_PASSWORD" + rm -f "$tarball" + mexec "rm -rf /root/maint-pkg && mkdir -p /root/maint-pkg && tar -C /root/maint-pkg -xzf /tmp/maint-src.tgz && rm /tmp/maint-src.tgz" \ + >> "$LOGFILE" 2>&1 || fatal "maint tree extract failed" + + step "Installing thresholds TOML" + copy_to_vm "$THRESHOLDS_TOML" "/tmp/maintenance-thresholds.toml" \ + "$ROOT_PASSWORD" + mexec "mkdir -p /root/.config/archsetup && mv /tmp/maintenance-thresholds.toml /root/.config/archsetup/maintenance-thresholds.toml" \ + >> "$LOGFILE" 2>&1 || fatal "TOML install failed" + + step "Smoke: maint status --json runs" + mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \ + || fatal "maint status failed in the VM" + success "Bootstrap complete ($(python3 -c " +import json; print(len(json.load(open('$RESULTS_DIR/bootstrap-status.json'))['metrics']))") metrics probed)" +} + +# ─── Execution ─────────────────────────────────────────────────────── + +PASS=() FAIL=() + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # Sourced in the runner's shell so break/fix/assert see the helpers; + # unset afterwards so the next scenario can't inherit stale functions. + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} + +section "Preparing VM" +stop_qemu 2>/dev/null || true +step "Restoring clean-install snapshot" +restore_snapshot "$DISK_PATH" "clean-install" || fatal "restore failed" +boot_vm +bootstrap_vm + +step "Freezing bootstrap as snapshot: $READY_SNAPSHOT" +stop_qemu +snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT" \ + && delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" +create_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "snapshot failed" + +first_group=true +for g in "${GROUP_ORDER[@]}"; do + section "Group: $g" + if [ "$first_group" = "true" ]; then + first_group=false + else + step "Isolation boundary: restore $READY_SNAPSHOT" + stop_qemu + restore_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "restore failed" + fi + boot_vm + for i in "${!S_FILES[@]}"; do + [ "${S_GROUPS[$i]}" = "$g" ] && run_scenario "$i" + done + stop_qemu +done + +# ─── Report ────────────────────────────────────────────────────────── + +section "Results" +for n in "${PASS[@]}"; do success "PASS $n"; done +for n in "${FAIL[@]}"; do error "FAIL $n"; done +info "Log: $LOGFILE" + +if [ "$KEEP_VM" = "true" ]; then + warn "--keep: VM left in post-run state; base image NOT restored" + warn "restore by hand: qemu-img snapshot -a clean-install $DISK_PATH" + CLEANUP_DONE=1 +else + cleanup_scenarios +fi + +if [ ${#FAIL[@]} -gt 0 ]; then + error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed" + exit 1 +fi +success "All ${#PASS[@]} scenarios passed" +exit 0 diff --git a/scripts/testing/run-test-baremetal.sh b/scripts/testing/run-test-baremetal.sh index ae88316..d22c424 100755 --- a/scripts/testing/run-test-baremetal.sh +++ b/scripts/testing/run-test-baremetal.sh @@ -20,13 +20,16 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" # Source utilities source "$SCRIPT_DIR/lib/logging.sh" -source "$SCRIPT_DIR/lib/validation.sh" +source "$SCRIPT_DIR/lib/validation.sh" # live helpers: ssh_cmd, capture_*_state, analyze_log_diff, generate_issue_report +source "$SCRIPT_DIR/lib/vm-utils.sh" # inject_root_key + SSH_OPTS/SSH_KEY_OPT for key auth +source "$SCRIPT_DIR/lib/testinfra.sh" # run_testinfra_validation (authoritative validator) # Parse arguments ROLLBACK_FIRST=false ROLLBACK_AFTER=false TARGET_HOST="" ROOT_PASSWORD="" +PORT="22" usage() { echo "Usage: $0 --host <hostname> --password <root_password> [options]" @@ -36,6 +39,7 @@ usage() { echo " --password <password> Root password for SSH" echo "" echo "Options:" + echo " --port <port> SSH port (default 22; use 2222 to target a test VM)" echo " --rollback-first Roll back to genesis snapshots before running" echo " --rollback-after Roll back to genesis snapshots after test (cleanup)" echo " --validate-only Skip archsetup, only run validation checks" @@ -55,6 +59,10 @@ while [[ $# -gt 0 ]]; do ROOT_PASSWORD="${2:?--password requires a value}" shift 2 ;; + --port) + PORT="${2:?--port requires a value}" + shift 2 + ;; --rollback-first) ROLLBACK_FIRST=true shift @@ -94,9 +102,16 @@ cleanup_baremetal() { } trap cleanup_baremetal EXIT -# Override VM_IP for validation.sh ssh_cmd function -# shellcheck disable=SC2034 # consumed by the sourced validation.sh +# Connection globals consumed by ssh_cmd (validation.sh), inject_root_key +# (vm-utils.sh), and run_testinfra_validation (testinfra.sh). +# shellcheck disable=SC2034 # consumed by the sourced libraries VM_IP="$TARGET_HOST" +# shellcheck disable=SC2034 +SSH_PORT="$PORT" +# Test-user source for testinfra (reads USERNAME); the bare-metal user is the +# archsetup default, cjennings, same as the VM conf. +# shellcheck disable=SC2034 +ARCHSETUP_VM_CONF="$SCRIPT_DIR/archsetup-vm.conf" # Initialize logging mkdir -p "$TEST_RESULTS_DIR" @@ -109,8 +124,8 @@ info "Target: $TARGET_HOST" # Test SSH connectivity step "Testing SSH connectivity to $TARGET_HOST" if ! sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - -o ConnectTimeout=10 "root@$TARGET_HOST" "echo connected" &>/dev/null; then - fatal "Cannot connect to $TARGET_HOST via SSH" + -o ConnectTimeout=10 -p "$PORT" "root@$TARGET_HOST" "echo connected" &>/dev/null; then + fatal "Cannot connect to $TARGET_HOST:$PORT via SSH" fi success "SSH connection OK" @@ -147,6 +162,15 @@ if $ROLLBACK_FIRST; then success "Reconnected" fi +# Authorize a throwaway root key before archsetup hardens sshd. archsetup sets +# PermitRootLogin prohibit-password and reloads sshd partway through, which kills +# root *password* SSH; key auth survives it, so every later ssh_cmd and the +# Testinfra sweep keep working. Placed after any genesis rollback so the key +# isn't rolled away. Best-effort: a failure only risks the post-hardening steps. +step "Authorizing throwaway root key (survives sshd hardening)" +inject_root_key "$TEST_RESULTS_DIR/root_key" || \ + warn "Continuing without an injected root key — SSH may fail after archsetup hardens sshd" + if ! $VALIDATE_ONLY; then # Capture pre-install state capture_pre_install_state "$TEST_RESULTS_DIR" @@ -161,7 +185,7 @@ if ! $VALIDATE_ONLY; then step "Transferring to $TARGET_HOST" ssh_cmd "rm -rf /tmp/archsetup-test && mkdir -p /tmp/archsetup-test" sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "$BUNDLE_FILE" "root@$TARGET_HOST:/tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 + ${SSH_KEY_OPT:-} -P "$PORT" "$BUNDLE_FILE" "root@$TARGET_HOST:/tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 step "Extracting on target" ssh_cmd "cd /tmp && git clone /tmp/archsetup.bundle archsetup-test && rm /tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 @@ -223,12 +247,12 @@ if ! $VALIDATE_ONLY; then step "Copying archsetup log" sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$TARGET_HOST:/var/log/archsetup-*.log" "$TEST_RESULTS_DIR/" 2>> "$LOGFILE" || \ + ${SSH_KEY_OPT:-} -P "$PORT" "root@$TARGET_HOST:/var/log/archsetup-*.log" "$TEST_RESULTS_DIR/" 2>> "$LOGFILE" || \ warn "Could not copy archsetup log" step "Copying archsetup output" sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$TARGET_HOST:$REMOTE_LOG" "$TEST_RESULTS_DIR/archsetup-output.log" 2>> "$LOGFILE" || \ + ${SSH_KEY_OPT:-} -P "$PORT" "root@$TARGET_HOST:$REMOTE_LOG" "$TEST_RESULTS_DIR/archsetup-output.log" 2>> "$LOGFILE" || \ warn "Could not copy output log" # Capture post-install state @@ -239,13 +263,14 @@ else mkdir -p "$TEST_RESULTS_DIR/pre-install" "$TEST_RESULTS_DIR/post-install" fi -# Run validations -run_all_validations -validate_all_services - -# Additional ZFS-specific validations -section "ZFS-Specific Validations" -validate_zfs_services +# Run validations. Testinfra is the authoritative validator (same as the VM +# runner); its ZFS-conditional pytest checks cover what validate_zfs_services +# used to, and it connects over the key authorized above. set +e because it +# returns pytest's rc (non-zero on failures) and the report + summary below must +# still run; the verdict is carried by testinfra_rc and the exit code at the end. +set +e +run_testinfra_validation "$TEST_RESULTS_DIR" +testinfra_rc=$? # Analyze logs if we ran archsetup if ! $VALIDATE_ONLY; then @@ -255,8 +280,8 @@ fi # Generate reports generate_issue_report "$TEST_RESULTS_DIR" "$ARCHZFS_INBOX" -# Set validation result (TEST_PASSED is the boolean; VALIDATION_PASSED stays the counter) -if [ "$VALIDATION_FAILED" -eq 0 ]; then +# The run passes only if the Testinfra sweep passed. +if [ "$testinfra_rc" -eq 0 ]; then TEST_PASSED=true else TEST_PASSED=false diff --git a/scripts/testing/run-test.sh b/scripts/testing/run-test.sh index 9b47747..f962df3 100755 --- a/scripts/testing/run-test.sh +++ b/scripts/testing/run-test.sh @@ -24,6 +24,7 @@ source "$SCRIPT_DIR/lib/logging.sh" source "$SCRIPT_DIR/lib/vm-utils.sh" source "$SCRIPT_DIR/lib/network-diagnostics.sh" source "$SCRIPT_DIR/lib/validation.sh" +source "$SCRIPT_DIR/lib/testinfra.sh" # Parse arguments KEEP_VM=false @@ -49,6 +50,9 @@ while [[ $# -gt 0 ]]; do echo " --keep Keep VM in post-test state (for debugging)" echo " --script Specify custom archsetup script to test" echo " --snapshot Snapshot name to revert to (default: clean-install)" + echo "" + echo "Env: FS_PROFILE=btrfs|zfs (default btrfs) selects the base image" + echo " built by create-base-vm.sh. e.g. FS_PROFILE=zfs $0" exit 1 ;; esac @@ -99,6 +103,7 @@ init_logging "$LOGFILE" init_vm_paths "$VM_IMAGES_DIR" section "ArchSetup Test Run: $TIMESTAMP" +info "Filesystem profile: $FS_PROFILE (image: $(basename "$DISK_PATH"))" # Verify archsetup script exists if [ ! -f "$ARCHSETUP_SCRIPT" ]; then @@ -107,7 +112,11 @@ fi # Check disk exists if [ ! -f "$DISK_PATH" ]; then - info "Create it first: ./scripts/testing/create-base-vm.sh" + if [ "$FS_PROFILE" = "btrfs" ]; then + info "Create it first: ./scripts/testing/create-base-vm.sh" + else + info "Create it first: FS_PROFILE=$FS_PROFILE ./scripts/testing/create-base-vm.sh" + fi fatal "Base disk not found: $DISK_PATH" fi @@ -141,6 +150,13 @@ start_qemu "$DISK_PATH" "disk" "" "none" || fatal "Failed to start VM" wait_for_ssh "$ROOT_PASSWORD" 120 || fatal "VM SSH not available" stop_timer "boot" +# Authorize a root key now, before archsetup runs. archsetup hardens sshd to +# PermitRootLogin prohibit-password partway through, which breaks the harness's +# root password SSH; key auth survives it. Without this, the run aborts mid-way +# (before any validation) once the hardening step lands. +inject_root_key "$TEST_RESULTS_DIR/root_key" || \ + warn "Continuing without root key - run may break at the sshd hardening step" + # Run network diagnostics if ! run_network_diagnostics; then fatal "Network diagnostics failed - aborting test" @@ -241,7 +257,8 @@ fi # Poll for completion step "Monitoring archsetup progress (polling every 30 seconds)..." POLL_COUNT=0 -MAX_POLLS=180 # 90 minutes max (180 * 30 seconds) +MAX_POLLS=300 # 150 minutes max (300 * 30 seconds); a full install with heavy + # AUR builds (e.g. vagrant) can exceed 90 min on a slow mirror while [ $POLL_COUNT -lt $MAX_POLLS ]; do # Check if archsetup process is still running @@ -262,7 +279,7 @@ while [ $POLL_COUNT -lt $MAX_POLLS ]; do done if [ $POLL_COUNT -ge $MAX_POLLS ]; then - error "ArchSetup timed out after 90 minutes" + error "ArchSetup timed out after 150 minutes" ARCHSETUP_EXIT_CODE=124 else # Get exit code from the remote log @@ -308,18 +325,17 @@ copy_from_vm "/var/log/archsetup-installed-packages.txt" "$TEST_RESULTS_DIR/" "$ # Capture post-install state capture_post_install_state "$TEST_RESULTS_DIR" -# Run comprehensive validation -# This uses the validation.sh library for all checks. +# Run comprehensive validation (Testinfra/pytest is the primary validator; the +# old shell run_all_validations sweep was retired once pytest reached parity). # # From here to the end of the script, errexit is disabled on purpose: the -# validation functions are designed to fail-and-count (see VALIDATION_FAILED) -# rather than abort, and the analysis/report-generation steps below can also -# legitimately return non-zero. With `set -e` active, a single failed check -# would kill the run before the test report is written or the VM is cleaned -# up. Pass/fail is signalled explicitly by the exit code at the bottom. +# analysis/report-generation steps below can legitimately return non-zero, and +# with `set -e` active a single failed check would kill the run before the test +# report is written or the VM is cleaned up. Pass/fail is signalled explicitly +# by the exit code at the bottom. set +e -run_all_validations -validate_all_services +run_testinfra_validation "$TEST_RESULTS_DIR" +testinfra_rc=$? # Analyze log differences (pre vs post install) analyze_log_diff "$TEST_RESULTS_DIR" @@ -328,8 +344,8 @@ analyze_log_diff "$TEST_RESULTS_DIR" # If base install issues found and archzfs inbox exists, create issue file generate_issue_report "$TEST_RESULTS_DIR" "$ARCHZFS_INBOX" -# Set validation result based on failure count -if [ "$VALIDATION_FAILED" -eq 0 ]; then +# The run passes only if the Testinfra sweep passed. +if [ "$testinfra_rc" -eq 0 ]; then TEST_PASSED=true else TEST_PASSED=false diff --git a/scripts/testing/tests/conftest.py b/scripts/testing/tests/conftest.py new file mode 100644 index 0000000..680c967 --- /dev/null +++ b/scripts/testing/tests/conftest.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Pytest + Testinfra config for archsetup post-install validation. + +These tests run on the *host* and connect to the freshly-installed VM over SSH +(Testinfra provides the `host` fixture, parametrized from --hosts). This file +adds two things the bespoke shell harness had that Testinfra does not: + + - Failure attribution. Each check is marked with the layer that owns a + failure (archsetup | base_install | unknown), mirroring validation.sh's + attribute_issue. Failures are bucketed and written to --attribution-file + so run-test.sh can route base-install issues to the archzfs inbox as before. + - Tiering markers (smoke | integration) so `pytest -m smoke` is a fast gate. + +The `target_user` fixture supplies the account archsetup created; it reads +ARCHSETUP_TEST_USER (set by run-test.sh from the VM conf) and defaults to the +historical "cjennings". +""" + +import os + +import pytest + + +_ATTRIBUTION_BUCKETS = ("archsetup", "base_install", "unknown") +_failures = {bucket: [] for bucket in _ATTRIBUTION_BUCKETS} + + +def pytest_addoption(parser): + parser.addoption( + "--attribution-file", + action="store", + default=None, + help="write the failure attribution report (archsetup/base_install/unknown) here", + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "attribution(bucket): layer that owns a failure — archsetup, base_install, or unknown", + ) + config.addinivalue_line("markers", "smoke: fast subset (user, key packages, dotfiles present)") + config.addinivalue_line("markers", "integration: full post-install checks") + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport(item, call): + report = yield + if report.when == "call" and report.failed: + marker = item.get_closest_marker("attribution") + bucket = marker.args[0] if (marker and marker.args) else "archsetup" + if bucket not in _failures: + bucket = "unknown" + _failures[bucket].append(item.nodeid) + return report + + +def pytest_sessionfinish(session, exitstatus): + path = session.config.getoption("--attribution-file") + if not path: + return + with open(path, "w") as fh: + for bucket in _ATTRIBUTION_BUCKETS: + fh.write("[%s]\n" % bucket) + for nodeid in _failures[bucket]: + fh.write(" %s\n" % nodeid) + + +@pytest.fixture(scope="session") +def target_user(): + """The account archsetup created in the VM under test.""" + return os.environ.get("ARCHSETUP_TEST_USER", "cjennings") + + +@pytest.fixture(scope="session") +def home(target_user): + return "/home/%s" % target_user + + +@pytest.fixture(scope="module") +def zfs_root(host): + """True when the VM's root filesystem is ZFS (gates ZFS-specific checks).""" + return host.run("findmnt -n -o FSTYPE /").stdout.strip() == "zfs" + + +@pytest.fixture(scope="module") +def has_nvme(host): + """True when the VM exposes an NVMe device.""" + return host.run("ls /dev/nvme0n1 2>/dev/null").rc == 0 + + +@pytest.fixture(scope="module") +def hyprland_installed(host): + return host.package("hyprland").is_installed + + +@pytest.fixture(scope="module") +def dwm_installed(host): + return host.file("/usr/local/bin/dwm").exists + + +@pytest.fixture(scope="module") +def compositor_running(host): + """A graphical session is live (gates socket/portal checks that need one).""" + return host.run("pgrep -x Hyprland").rc == 0 + + +@pytest.fixture(scope="module") +def on_slirp(host): + """QEMU user-mode networking (10.0.2.x) — no multicast, so mDNS can't work.""" + return "10.0.2." in host.run("ip -4 addr show").stdout diff --git a/scripts/testing/tests/test_archsetup.py b/scripts/testing/tests/test_archsetup.py new file mode 100644 index 0000000..52fe3f7 --- /dev/null +++ b/scripts/testing/tests/test_archsetup.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: archsetup's own log and state markers. + +Parity port of validate_archsetup_log and validate_state_markers. +""" + +import pytest + + +EXPECTED_STATE_STEPS = 12 + + +@pytest.mark.attribution("archsetup") +def test_no_errors_in_archsetup_log(host): + out = host.run("grep -h '^Error:' /var/log/archsetup-*.log 2>/dev/null | wc -l") + count = int((out.stdout.strip() or "0")) + assert count == 0, "archsetup log reported %d Error: lines" % count + + +@pytest.mark.attribution("archsetup") +def test_all_install_steps_completed(host): + out = host.run("ls /var/lib/archsetup/state/ 2>/dev/null | wc -l") + count = int((out.stdout.strip() or "0")) + assert count >= EXPECTED_STATE_STEPS, ( + "only %d/%d install steps completed" % (count, EXPECTED_STATE_STEPS) + ) diff --git a/scripts/testing/tests/test_backups.py b/scripts/testing/tests/test_backups.py new file mode 100644 index 0000000..07da5ec --- /dev/null +++ b/scripts/testing/tests/test_backups.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: backup_system_file ran during a real install. + +Expansion coverage (P4). The unit suite (tests/backup-system-file/) covers the +helper's logic; this confirms it actually fires end-to-end — archsetup leaves a +<file>.archsetup.bak next to each pre-existing file it edits in place. + +These targets are edited unconditionally on every run (pacman.conf/makepkg.conf +always sed'd, sudoers always appended), so their backups must exist. +mkinitcpio.conf is edited only conditionally (the systemd-hook switch on +non-ZFS, or the nvme module on NVMe systems), so it gets its own fixture-gated +check below. Conditionally-edited files (locale.gen, geoclue, fstab) aren't +asserted here since their edits depend on the base image. +""" + +import pytest + + +ALWAYS_BACKED_UP = [ + "/etc/pacman.conf", + "/etc/makepkg.conf", + "/etc/sudoers", +] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("path", ALWAYS_BACKED_UP) +def test_backup_created_for_edited_file(host, path): + bak = host.file(path + ".archsetup.bak") + assert bak.exists, "%s.archsetup.bak missing — backup_system_file did not fire" % path + assert bak.is_file + + +@pytest.mark.attribution("archsetup") +def test_backup_created_for_mkinitcpio(host, zfs_root, has_nvme): + # archsetup edits /etc/mkinitcpio.conf only when it has something to change: + # the systemd-hook switch (non-ZFS only) or adding the nvme module (NVMe + # systems). A ZFS root with no NVMe touches neither, so there's no backup. + if zfs_root and not has_nvme: + pytest.skip("ZFS root + no NVMe: archsetup doesn't edit mkinitcpio.conf") + bak = host.file("/etc/mkinitcpio.conf.archsetup.bak") + assert bak.exists, \ + "/etc/mkinitcpio.conf.archsetup.bak missing — backup_system_file did not fire" + assert bak.is_file diff --git a/scripts/testing/tests/test_boot.py b/scripts/testing/tests/test_boot.py new file mode 100644 index 0000000..e442682 --- /dev/null +++ b/scripts/testing/tests/test_boot.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: boot, initramfs, and filesystem config. + +Parity port of validate_zfs_config, validate_boot_config, +validate_mkinitcpio_hooks, validate_initramfs_consolefont, validate_nvme_module. +Filesystem/hardware-specific checks are gated on fixtures. +""" + +import pytest + + +@pytest.mark.attribution("archsetup") +def test_bootloader_installed(host, zfs_root): + # A ZFS root boots via ZFSBootMenu (archangel installs the EFI binary under + # /efi/EFI/ZBM), so there is no GRUB; a non-ZFS root uses GRUB. + if zfs_root: + assert host.file("/efi/EFI/ZBM/zfsbootmenu.efi").exists, \ + "ZFS root must have the ZFSBootMenu EFI binary" + else: + assert host.file("/boot/grub/grub.cfg").exists, \ + "non-ZFS root must have a GRUB config" + + +@pytest.mark.attribution("archsetup") +def test_mkinitcpio_hooks(host, zfs_root): + hooks = host.run("grep '^HOOKS=' /etc/mkinitcpio.conf").stdout + if zfs_root: + # ZFS must use the udev hook; the systemd hook breaks a ZFS boot. + assert " udev" in hooks or "(udev" in hooks, "ZFS root must use the udev hook" + assert "systemd" not in hooks, "ZFS root must not use the systemd hook" + else: + # Non-ZFS: either hook is acceptable. + assert ("systemd" in hooks) or ("udev" in hooks) + + +@pytest.mark.attribution("archsetup") +def test_console_font_configured(host, zfs_root): + # archsetup sets FONT=ter-132n in /etc/vconsole.conf on every run. + assert host.file("/etc/vconsole.conf").contains("^FONT=ter-132n"), \ + "archsetup should set FONT=ter-132n in /etc/vconsole.conf" + # On non-ZFS it also rebuilds the initramfs (mkinitcpio -P) so the font is + # baked in for early boot. On ZFS that rebuild is skipped (the busybox ZFS + # hook is incompatible with the systemd-hook switch), so the font applies at + # the vconsole layer once userspace starts, not inside the initramfs. + if zfs_root: + return + # Pick the main initramfs (this fleet runs linux-lts, so the name is + # initramfs-linux-lts.img, not initramfs-linux.img); skip the fallback image. + img = host.run( + "ls /boot/initramfs-*.img 2>/dev/null | grep -v fallback | head -1" + ).stdout.strip() + assert img, "no initramfs image found under /boot" + out = host.run("lsinitcpio %s 2>/dev/null | grep -cE 'consolefont.psf|ter-'" % img) + assert int((out.stdout.strip() or "0")) > 0, "console font not found in %s" % img + + +def test_nvme_module_when_nvme_present(host, has_nvme): + if not has_nvme: + pytest.skip("no NVMe device present") + modules = host.run("grep '^MODULES=' /etc/mkinitcpio.conf").stdout + assert "nvme" in modules, "NVMe system should list nvme in mkinitcpio MODULES" + + +def test_zfs_has_sanoid(host): + if not host.exists("zfs"): + pytest.skip("ZFS not installed (non-ZFS system)") + assert host.exists("sanoid"), "ZFS system should have sanoid installed" + + +def test_zfs_pre_pacman_snapshot_hook(host): + # archsetup installs a PreTransaction pacman hook + a self-pruning script so + # every pacman transaction is preceded by a rollback snapshot (configure_ + # pre_pacman_snapshots, run late in boot_ux). ZFS-root only. + if not host.exists("zfs"): + pytest.skip("ZFS not installed (non-ZFS system)") + script = host.file("/usr/local/bin/zfs-pre-snapshot") + assert script.exists and script.is_file, "pre-pacman snapshot script missing" + assert script.mode & 0o111, "pre-pacman snapshot script is not executable" + hook = host.file("/etc/pacman.d/hooks/zfs-snapshot.hook") + assert hook.exists and hook.is_file, "zfs-snapshot.hook missing" + assert "PreTransaction" in hook.content_string, "hook not PreTransaction" + assert "/usr/local/bin/zfs-pre-snapshot" in hook.content_string, \ + "hook does not exec the snapshot script" diff --git a/scripts/testing/tests/test_config_applied.py b/scripts/testing/tests/test_config_applied.py new file mode 100644 index 0000000..00c410e --- /dev/null +++ b/scripts/testing/tests/test_config_applied.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: archsetup's in-place config edits actually took effect. + +Expansion coverage (P4). These assert the *content* archsetup writes, not just +that a service is enabled — catching cases where a sed silently no-ops (e.g. +ParallelDownloads, which current Arch ships uncommented so a "^#"-only match +left it at 5). +""" + +import pytest + + +@pytest.mark.attribution("archsetup") +def test_pacman_parallel_downloads(host): + line = host.run("grep -E '^ParallelDownloads' /etc/pacman.conf").stdout + assert "ParallelDownloads = 10" in line, "ParallelDownloads not set to 10 (got: %r)" % line + + +@pytest.mark.attribution("archsetup") +def test_pacman_color_enabled(host): + assert host.run("grep -qx Color /etc/pacman.conf").rc == 0 + + +@pytest.mark.attribution("archsetup") +def test_pacman_multilib_enabled(host): + # -F: [multilib] is a literal section header, not a regex character class. + assert host.run("grep -Fxq '[multilib]' /etc/pacman.conf").rc == 0 + + +@pytest.mark.attribution("archsetup") +def test_makepkg_parallel_make(host): + line = host.run("grep -E '^MAKEFLAGS' /etc/makepkg.conf").stdout + assert "-j" in line, "MAKEFLAGS not configured for parallel make (got: %r)" % line + + +@pytest.mark.attribution("archsetup") +def test_makepkg_options_trimmed(host): + opts = host.run("grep -E '^OPTIONS' /etc/makepkg.conf").stdout + assert "!debug" in opts and "purge" in opts, "makepkg OPTIONS not customized" + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("rel", ["dns.conf", "wifi-privacy.conf"]) +def test_networkmanager_dropin(host, rel): + assert host.file("/etc/NetworkManager/conf.d/%s" % rel).exists + + +@pytest.mark.attribution("archsetup") +def test_fail2ban_jail_local(host): + assert host.file("/etc/fail2ban/jail.local").exists + + +@pytest.mark.attribution("archsetup") +def test_reflector_config(host): + assert host.file("/etc/xdg/reflector/reflector.conf").exists diff --git a/scripts/testing/tests/test_desktop.py b/scripts/testing/tests/test_desktop.py new file mode 100644 index 0000000..6f79bfd --- /dev/null +++ b/scripts/testing/tests/test_desktop.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: window manager + desktop integration. + +Parity port of validate_window_manager and its Hyprland/DWM branches, plus +validate_autologin_config. Hyprland and DWM checks are gated on which DE the +run installed; socket/portal-query checks are gated on a live compositor (the +headless test VM has none). + +Note: validate_hyprland_tools historically checked `swww`, but archsetup now +installs `awww` (swww successor) and `pacman -Q swww` no longer matches — so +this checks awww. That divergence from the shell sweep is a correctness fix. +""" + +import pytest + + +HYPRLAND_TOOLS = [ + "hyprland", "hypridle", "hyprlock", "waybar", "fuzzel", + "awww", "grim", "slurp", "gammastep", "foot", +] + +HYPRLAND_CONFIGS = [ + ".config/hypr/hyprland.conf", + ".config/hypr/hypridle.conf", + ".config/hypr/hyprlock.conf", + ".config/waybar/config", + ".config/fuzzel/fuzzel.ini", + ".config/gammastep/config.ini", +] + +SUCKLESS_TOOLS = ["dwm", "st", "dmenu", "slock"] + +PORTALS_CONF = ".config/xdg-desktop-portal/portals.conf" + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", HYPRLAND_TOOLS) +def test_hyprland_tool_installed(host, hyprland_installed, pkg): + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + assert host.package(pkg).is_installed, "%s missing" % pkg + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("rel", HYPRLAND_CONFIGS) +def test_hyprland_config_present(host, hyprland_installed, home, rel): + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + assert host.file("%s/%s" % (home, rel)).exists, "%s missing" % rel + + +@pytest.mark.attribution("archsetup") +def test_live_update_guard_installed(host, hyprland_installed): + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + guard = host.file("/usr/local/bin/hypr-live-update-guard") + assert guard.exists, "live-update guard script missing" + assert guard.mode & 0o111, "live-update guard not executable" + hook = host.file("/etc/pacman.d/hooks/hypr-live-update-guard.hook") + assert hook.exists, "live-update guard pacman hook missing" + assert "hypr-live-update-guard" in hook.content_string, \ + "hook does not invoke the guard script" + + +@pytest.mark.attribution("archsetup") +def test_portal_settings_backend_not_disabled(host, hyprland_installed, home): + if not hyprland_installed: + pytest.skip("Hyprland not installed") + conf = host.file("%s/%s" % (home, PORTALS_CONF)) + assert conf.exists, "portals.conf missing" + line = host.run( + "grep org.freedesktop.impl.portal.Settings %s" % conf.path + ).stdout + assert "=none" not in line.replace(" ", ""), "Settings portal disabled (=none)" + + +def test_portal_returns_dark_mode(host, hyprland_installed, compositor_running, target_user): + if not hyprland_installed: + pytest.skip("Hyprland not installed") + if not compositor_running: + pytest.skip("no compositor running (headless) — portal query not applicable") + cmd = ( + "sudo -u %s busctl --user call org.freedesktop.portal.Desktop " + "/org/freedesktop/portal/desktop org.freedesktop.portal.Settings Read " + "'ss' 'org.freedesktop.appearance' 'color-scheme'" % target_user + ) + out = host.run(cmd).stdout + assert "u 1" in out, "Settings portal should report color-scheme=1 (dark)" + + +def test_hyprland_socket(host, hyprland_installed, compositor_running): + if not hyprland_installed: + pytest.skip("Hyprland not installed") + if not compositor_running: + pytest.skip("Hyprland not running (headless) — socket check not applicable") + assert host.run("test -S /tmp/hypr/*/.socket.sock").rc == 0 + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("tool", SUCKLESS_TOOLS) +def test_suckless_tool_installed(host, dwm_installed, tool): + if not dwm_installed: + pytest.skip("DWM not installed (DESKTOP_ENV != dwm)") + assert host.file("/usr/local/bin/%s" % tool).exists, "%s missing" % tool + + +def test_autologin_configured(host): + conf = host.file("/etc/systemd/system/getty@tty1.service.d/autologin.conf") + if not conf.exists: + pytest.skip("autologin not configured (AUTOLOGIN=no, may be intentional)") + assert conf.exists + + +BT_PANEL_BINS = ["bt", "bt-panel", "bt-priv", "waybar-bt"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("name", BT_PANEL_BINS) +def test_bt_panel_bin_stowed(host, hyprland_installed, home, name): + # Executable via either stow shape (per-file symlink or folded dir). + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + path = "%s/.local/bin/%s" % (home, name) + assert host.file(path).exists, "%s missing from ~/.local/bin" % name + assert host.run("test -x %s" % path).rc == 0, "%s not executable" % name + + +@pytest.mark.attribution("archsetup") +def test_bt_panel_wired(host, hyprland_installed, home): + # A fresh install lands the panel reachable: bar module, keybind, css. + if not hyprland_installed: + pytest.skip("Hyprland not installed (DESKTOP_ENV != hyprland)") + waybar = host.file("%s/.config/waybar/config" % home) + assert "custom/bluetooth" in waybar.content_string, \ + "waybar config lacks the custom/bluetooth module" + hyprconf = host.file("%s/.config/hypr/hyprland.conf" % home) + assert "bt-panel" in hyprconf.content_string, \ + "hyprland.conf lacks the bt-panel keybind" + assert host.file("%s/.config/themes/dupre/panel.css" % home).exists, \ + "shared panel css missing from the stowed theme" diff --git a/scripts/testing/tests/test_dotfiles.py b/scripts/testing/tests/test_dotfiles.py new file mode 100644 index 0000000..cd6e474 --- /dev/null +++ b/scripts/testing/tests/test_dotfiles.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: dotfiles stowed for the user. + +Parity port of validate_dotfiles from validation.sh: .zshrc must be a symlink +into the ~/.dotfiles stow tree, not broken, and readable by the user (not just +root). +""" + +import pytest + + +@pytest.mark.attribution("archsetup") +def test_zshrc_stowed_and_readable(host, target_user): + zshrc = host.file("/home/%s/.zshrc" % target_user) + assert zshrc.is_symlink, ".zshrc should be a stow symlink" + assert ".dotfiles/" in zshrc.linked_to, "symlink should point into ~/.dotfiles" + assert zshrc.exists, "symlink target must exist (not broken)" + # Readable by the user, not only root. + assert host.run("sudo -u %s test -r %s" % (target_user, zshrc.path)).rc == 0 diff --git a/scripts/testing/tests/test_hardening.py b/scripts/testing/tests/test_hardening.py new file mode 100644 index 0000000..f12b0e6 --- /dev/null +++ b/scripts/testing/tests/test_hardening.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: security/system hardening archsetup applies. + +Expansion coverage (P4) — these were not in the original shell sweep. They +assert the system-level changes archsetup makes in place: sshd root hardening, +quiet kernel console, an emptied /etc/issue, the console font, and the EFI +mount permission tightening. +""" + +import pytest + + +@pytest.mark.smoke +@pytest.mark.attribution("archsetup") +def test_sshd_root_prohibit_password(host): + conf = host.file("/etc/ssh/sshd_config.d/10-hardening.conf") + assert conf.exists, "sshd hardening drop-in missing" + assert "PermitRootLogin prohibit-password" in conf.content_string + + +@pytest.mark.attribution("archsetup") +def test_quiet_printk_sysctl(host): + conf = host.file("/etc/sysctl.d/20-quiet-printk.conf") + assert conf.exists + assert "kernel.printk" in conf.content_string + + +@pytest.mark.attribution("archsetup") +def test_issue_emptied(host): + # archsetup truncates /etc/issue to drop the distro/date banner. + assert host.file("/etc/issue").size == 0 + + +@pytest.mark.attribution("archsetup") +def test_console_font_configured(host): + assert "ter-132n" in host.file("/etc/vconsole.conf").content_string + + +@pytest.mark.attribution("archsetup") +def test_efi_mount_permissions_tightened(host): + # archsetup adds fmask/dmask to the /efi vfat line so it isn't world-readable. + fstab = host.file("/etc/fstab").content_string + efi_lines = [ + ln for ln in fstab.splitlines() + if ln.strip() and not ln.lstrip().startswith("#") + and " /efi " in ln and " vfat " in ln + ] + if not efi_lines: + pytest.skip("no /efi vfat line in fstab") + assert all("fmask=" in ln for ln in efi_lines), "/efi mount not permission-tightened" diff --git a/scripts/testing/tests/test_keyring.py b/scripts/testing/tests/test_keyring.py new file mode 100644 index 0000000..99d322d --- /dev/null +++ b/scripts/testing/tests/test_keyring.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: gnome-keyring pre-configuration. + +Parity port of validate_gnome_keyring_setup: the keyrings dir must exist, be +mode 700, owned by the user, with the default keyring set to "login". +""" + +import pytest + + +@pytest.fixture(scope="session") +def keyring_dir(home): + return "%s/.local/share/keyrings" % home + + +@pytest.mark.attribution("archsetup") +def test_keyring_dir_exists(host, keyring_dir): + assert host.file(keyring_dir).is_directory + + +@pytest.mark.attribution("archsetup") +def test_keyring_dir_mode_700(host, keyring_dir): + assert host.file(keyring_dir).mode == 0o700 + + +@pytest.mark.attribution("archsetup") +def test_keyring_dir_owned_by_user(host, keyring_dir, target_user): + assert host.file(keyring_dir).user == target_user + + +@pytest.mark.attribution("archsetup") +def test_default_keyring_is_login(host, keyring_dir): + default = host.file("%s/default" % keyring_dir) + assert default.exists, "default keyring file missing" + assert default.content_string.strip() == "login" diff --git a/scripts/testing/tests/test_packages.py b/scripts/testing/tests/test_packages.py new file mode 100644 index 0000000..e0387d6 --- /dev/null +++ b/scripts/testing/tests/test_packages.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: package managers and key packages. + +Parity port of validate_yay_installed, validate_pacman_working, +validate_terminus_font, validate_emacs, validate_git_config, validate_dev_tools. +""" + +import pytest + + +DEV_TOOLS = ["python", "node", "npm", "go", "rustc"] + + +@pytest.mark.smoke +@pytest.mark.attribution("archsetup") +def test_yay_installed(host): + assert host.exists("yay"), "yay binary not on PATH" + + +@pytest.mark.attribution("archsetup") +def test_yay_functional(host, target_user): + # yay must actually query the package DB as the user, not just exist. + assert host.run("sudo -u %s yay -Qi yay" % target_user).rc == 0 + + +@pytest.mark.smoke +@pytest.mark.attribution("unknown") +def test_pacman_functional(host): + assert host.package("base").is_installed + + +@pytest.mark.attribution("archsetup") +def test_terminus_font_installed(host): + assert host.package("terminus-font").is_installed + + +@pytest.mark.attribution("archsetup") +def test_emacs_installed(host): + assert host.exists("emacs") + + +@pytest.mark.attribution("archsetup") +def test_emacs_config_readable_by_user(host, target_user, home): + emacsd = host.file("%s/.emacs.d" % home) + if not emacsd.exists: + pytest.skip(".emacs.d not present (config dir optional on some profiles)") + assert emacsd.is_directory + assert host.run("sudo -u %s ls %s" % (target_user, emacsd.path)).rc == 0 + + +@pytest.mark.smoke +@pytest.mark.attribution("archsetup") +def test_git_installed(host): + assert host.exists("git") + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("tool", DEV_TOOLS) +def test_dev_tool_present(host, tool): + assert host.exists(tool), "dev tool %s missing from PATH" % tool + + +BLUETOOTH_STACK = ["bluez", "bluez-utils"] +VPN_STACK = ["wireguard-tools", "proton-vpn-cli", "tailscale"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", BLUETOOTH_STACK) +def test_bluetooth_stack_installed(host, pkg): + assert host.package(pkg).is_installed + + +# bt panel replaced blueman; zoom-web replaced zoom; the net panel's Tunnels +# view + proton-vpn-cli replaced the GTK app (they can't run concurrently). +RETIRED_PACKAGES = ["blueman", "zoom", "proton-vpn-gtk-app"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", RETIRED_PACKAGES) +def test_retired_package_not_installed(host, pkg): + # A reappearance means an install step regressed. + assert not host.package(pkg).is_installed + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("pkg", VPN_STACK) +def test_vpn_stack_installed(host, pkg): + assert host.package(pkg).is_installed + + +@pytest.mark.attribution("archsetup") +def test_tailscale_operator_granted(host, target_user): + # The installer grants operator so the net panel can toggle tailscale + # without sudo. Prefs only answer when the daemon is up. + if not host.service("tailscaled").is_running: + pytest.skip("tailscaled not running") + out = host.run("tailscale debug prefs") + assert out.rc == 0, "tailscale debug prefs failed" + assert '"OperatorUser": "%s"' % target_user in out.stdout + + +@pytest.mark.attribution("archsetup") +def test_eask_installed_user_local(host, home): + # Installed via npm -g --prefix ~/.local as the user; chime and + # linear-emacs shell out to it. + f = host.file("%s/.local/bin/eask" % home) + assert f.exists, "eask missing from ~/.local/bin" + npmrc = host.file("%s/.npmrc" % home) + assert npmrc.exists, ".npmrc (user npm prefix) not stowed" diff --git a/scripts/testing/tests/test_services.py b/scripts/testing/tests/test_services.py new file mode 100644 index 0000000..0ca3970 --- /dev/null +++ b/scripts/testing/tests/test_services.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: services, timers, and their functional health. + +Parity port of validate_firewall, validate_dns_config, validate_avahi, +validate_fail2ban, validate_networkmanager, and validate_all_services / +validate_service_functions. + +Mapping of the shell sweep's three outcomes: + - validation_fail (hard) -> assert + - validation_warn (soft) -> pytest.skip with the reason (visible, never red) + - validation_skip (precond)-> pytest.skip gated on a fixture +""" + +import pytest + + +# Required services: (name, must_be_active). ufw can't activate in the VM (no +# iptables kernel modules), so it's enabled-only; cronie/atd are enabled-only too. +REQUIRED_ENABLED_ACTIVE = ["sshd", "systemd-resolved", "fail2ban", "NetworkManager", "rngd"] +REQUIRED_ENABLED_ONLY = ["ufw", "cronie", "atd"] +REQUIRED_TIMERS = ["reflector.timer", "paccache.timer"] +OPTIONAL_SERVICES = ["avahi-daemon", "bluetooth", "cups", "docker", "tailscaled"] + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("svc", REQUIRED_ENABLED_ACTIVE) +def test_required_service_enabled_and_active(host, svc): + s = host.service(svc) + assert s.is_enabled, "%s should be enabled" % svc + assert s.is_running, "%s should be active" % svc + + +@pytest.mark.smoke +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("svc", REQUIRED_ENABLED_ONLY) +def test_required_service_enabled(host, svc): + assert host.service(svc).is_enabled, "%s should be enabled" % svc + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("timer", REQUIRED_TIMERS) +def test_required_timer_enabled(host, timer): + assert host.service(timer).is_enabled, "%s should be enabled" % timer + + +@pytest.mark.parametrize("svc", OPTIONAL_SERVICES) +def test_optional_service(host, svc): + # Optional: warn-if-missing in the shell sweep -> skip here so it never reds. + if not host.service(svc).is_enabled: + pytest.skip("%s not enabled (optional)" % svc) + + +@pytest.mark.attribution("archsetup") +def test_dns_over_tls_dropin_present(host): + # archsetup ships /etc/systemd/resolved.conf.d/dns-over-tls.conf. + assert host.file("/etc/systemd/resolved.conf.d/dns-over-tls.conf").exists + + +@pytest.mark.attribution("archsetup") +def test_fail2ban_responds(host): + assert host.run("fail2ban-client status").rc == 0 + + +@pytest.mark.attribution("archsetup") +def test_networkmanager_responds(host): + assert host.run("nmcli general status").rc == 0 + + +@pytest.mark.attribution("archsetup") +def test_log_cleanup_cron_installed(host, target_user): + out = host.run("sudo -u %s crontab -l" % target_user).stdout + assert "log-cleanup" in out, "log-cleanup entry missing from user crontab" + + +@pytest.mark.attribution("archsetup") +def test_syncthing_user_lingering_enabled(host, target_user): + # syncthing runs as a user service; lingering must be on for autostart. + assert host.file("/var/lib/systemd/linger/%s" % target_user).exists + + +def test_dns_resolution(host): + # Network-dependent; advisory in the shell sweep. Skip on failure. + if host.run("resolvectl query archlinux.org").rc != 0: + pytest.skip("DNS resolution query failed (network-dependent)") + + +def test_mdns_resolves(host, on_slirp): + # mDNS needs multicast, which QEMU slirp doesn't pass. + if on_slirp: + pytest.skip("mDNS not possible on slirp networking (no multicast)") + if not host.service("avahi-daemon").is_enabled: + pytest.skip("avahi-daemon not enabled") + hostname = host.run("hostname").stdout.strip() + assert host.run("ping -c 1 -W 2 %s.local" % hostname).rc == 0 + + +def test_docker_functional(host): + if not host.service("docker").is_enabled: + pytest.skip("docker not enabled") + if not host.service("docker").is_running: + # archsetup enables docker for next boot, not --now; pre-reboot this is correct. + pytest.skip("docker enabled but not started (starts on boot by design)") + assert host.run("docker info").rc == 0 diff --git a/scripts/testing/tests/test_users.py b/scripts/testing/tests/test_users.py new file mode 100644 index 0000000..c0097ed --- /dev/null +++ b/scripts/testing/tests/test_users.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Post-install checks: the user account archsetup creates. + +Parity port of validate_user_created / validate_user_shell / validate_user_groups. +""" + +import pytest + + +# Groups archsetup adds: wheel (useradd -G), the usermod -aG set, and docker +# (added later in the developer-workstation step). +EXPECTED_GROUPS = [ + "wheel", "sys", "adm", "network", "scanner", "power", "uucp", + "audio", "lp", "rfkill", "video", "storage", "optical", "users", "docker", +] + + +@pytest.mark.smoke +@pytest.mark.attribution("archsetup") +def test_user_exists(host, target_user): + assert host.user(target_user).exists + + +@pytest.mark.attribution("archsetup") +def test_user_shell_is_zsh(host, target_user): + # archsetup may set either path depending on how zsh resolves. + assert host.user(target_user).shell in ("/bin/zsh", "/usr/bin/zsh") + + +@pytest.mark.attribution("archsetup") +@pytest.mark.parametrize("group", EXPECTED_GROUPS) +def test_user_in_group(host, target_user, group): + # Parametrized so a failure names the exact missing group. + assert group in host.user(target_user).groups |