diff options
Diffstat (limited to 'scripts/testing')
| -rw-r--r-- | scripts/testing/README.org | 508 | ||||
| -rw-r--r-- | scripts/testing/archinstall-config.json | 117 | ||||
| -rwxr-xr-x | scripts/testing/cleanup-tests.sh | 171 | ||||
| -rwxr-xr-x | scripts/testing/create-base-vm.sh | 173 | ||||
| -rwxr-xr-x | scripts/testing/debug-vm.sh | 133 | ||||
| -rwxr-xr-x | scripts/testing/finalize-base-vm.sh | 31 | ||||
| -rwxr-xr-x | scripts/testing/lib/finalize-base-vm.sh | 21 | ||||
| -rwxr-xr-x | scripts/testing/lib/logging.sh | 151 | ||||
| -rw-r--r-- | scripts/testing/lib/network-diagnostics.sh | 60 | ||||
| -rw-r--r-- | scripts/testing/lib/validation.sh | 1080 | ||||
| -rwxr-xr-x | scripts/testing/lib/vm-utils.sh | 321 | ||||
| -rwxr-xr-x | scripts/testing/run-test-baremetal.sh | 310 | ||||
| -rwxr-xr-x | scripts/testing/run-test.sh | 367 | ||||
| -rwxr-xr-x | scripts/testing/setup-testing-env.sh | 191 |
14 files changed, 3634 insertions, 0 deletions
diff --git a/scripts/testing/README.org b/scripts/testing/README.org new file mode 100644 index 0000000..a52dbea --- /dev/null +++ b/scripts/testing/README.org @@ -0,0 +1,508 @@ +#+TITLE: ArchSetup Testing Infrastructure +#+AUTHOR: Craig Jennings +#+DATE: 2025-11-08 + +* Overview + +This directory contains the complete testing infrastructure for archsetup, built on QEMU/KVM virtualization. It provides automated, reproducible testing of the archsetup installation script in isolated virtual machines. + +** Philosophy + +*Realism over speed.* We use full VMs (not containers) to test everything archsetup does: user creation, systemd services, X11/DWM, hardware drivers, and boot process. + +* Quick Start + +** One-Time Setup + +#+begin_src bash +# Install required packages and configure environment +./scripts/testing/setup-testing-env.sh + +# Log out and back in (for libvirt group membership) + +# Create the base VM (minimal Arch installation) +./scripts/testing/create-base-vm.sh +# Follow on-screen instructions to complete installation +# Run finalize-base-vm.sh when done +#+end_src + +** Run a Test + +#+begin_src bash +# Test the current archsetup script +./scripts/testing/run-test.sh + +# Test a specific version +./scripts/testing/run-test.sh --script /path/to/archsetup + +# Keep VM running for debugging +./scripts/testing/run-test.sh --keep +#+end_src + +** Debug Interactively + +#+begin_src bash +# Clone base VM for debugging +./scripts/testing/debug-vm.sh + +# Use existing test disk +./scripts/testing/debug-vm.sh test-results/20251108-143000/test.qcow2 + +# Use base VM (read-only) +./scripts/testing/debug-vm.sh --base +#+end_src + +** Clean Up + +#+begin_src bash +# Interactive cleanup (prompts for confirmation) +./scripts/testing/cleanup-tests.sh + +# Keep last 10 test results +./scripts/testing/cleanup-tests.sh --keep 10 + +# Force cleanup without prompts +./scripts/testing/cleanup-tests.sh --force +#+end_src + +* Scripts + +** setup-testing-env.sh + +*Purpose:* One-time setup of testing infrastructure + +*What it does:* +- Installs QEMU/KVM, libvirt, and related tools +- Configures libvirt networking +- Adds user to libvirt group +- Verifies KVM support +- Creates directories for artifacts + +*When to run:* Once per development machine + +*Usage:* +#+begin_src bash +./scripts/testing/setup-testing-env.sh +#+end_src + +** create-base-vm.sh + +*Purpose:* Create the "golden image" minimal Arch VM + +*What it does:* +- Downloads latest Arch ISO +- Creates VM and boots from ISO +- Opens virt-viewer for you to complete installation manually + +*When to run:* Once (or when you want to refresh base image) + +*Usage:* +#+begin_src bash +./scripts/testing/create-base-vm.sh +#+end_src + +*Process:* +1. Script creates VM and boots from Arch ISO +2. virt-viewer opens automatically showing VM display +3. You complete installation manually using archinstall: + - Login as root (no password) + - Run =archinstall= + - Configure: hostname=archsetup-test, root password=archsetup + - Install packages: openssh git vim sudo + - Enable services: sshd, dhcpcd + - Configure SSH to allow root login + - Poweroff when done +4. Run =./scripts/testing/finalize-base-vm.sh= to complete + +*See also:* [[file:../../docs/base-vm-installation-checklist.org][Base VM Installation Checklist]] + +*Result:* Base VM image at =vm-images/archsetup-base.qcow2= + +** run-test.sh + +*Purpose:* Execute a full test run of archsetup + +*What it does:* +- Clones base VM (copy-on-write, fast) +- Starts test VM +- Transfers archsetup script and dotfiles +- Executes archsetup inside VM +- Captures logs and results +- Runs validation checks +- Generates test report +- Cleans up (unless =--keep=) + +*When to run:* Every time you want to test archsetup + +*Usage:* +#+begin_src bash +# Test current archsetup +./scripts/testing/run-test.sh + +# Test specific version +./scripts/testing/run-test.sh --script /path/to/archsetup + +# Keep VM for debugging +./scripts/testing/run-test.sh --keep +#+end_src + +*Time:* 30-60 minutes (mostly downloading packages) + +*Results:* Saved to =test-results/TIMESTAMP/= +- =test.log= - Complete log output +- =test-report.txt= - Summary of results +- =archsetup-*.log= - Log from archsetup script +- =*.txt= - Package lists from VM + +** debug-vm.sh + +*Purpose:* Launch VM for interactive debugging + +*What it does:* +- Creates VM from base image or existing test disk +- Configures console and SSH access +- Provides connection instructions + +*When to run:* When you need to manually test or debug + +*Usage:* +#+begin_src bash +# Clone base VM for debugging +./scripts/testing/debug-vm.sh + +# Use existing test disk +./scripts/testing/debug-vm.sh vm-images/archsetup-test-20251108-143000.qcow2 + +# Use base VM (read-only) +./scripts/testing/debug-vm.sh --base +#+end_src + +*Connect via:* +- Console: =virsh console archsetup-debug-TIMESTAMP= +- SSH: =ssh root@IP_ADDRESS= (password: archsetup) +- VNC: =virt-viewer archsetup-debug-TIMESTAMP= + +** cleanup-tests.sh + +*Purpose:* Clean up old test VMs and artifacts + +*What it does:* +- Lists all test VMs and destroys them +- Removes test disk images +- Keeps last N test results, deletes rest + +*When to run:* Periodically to free disk space + +*Usage:* +#+begin_src bash +# Interactive cleanup +./scripts/testing/cleanup-tests.sh + +# Keep last 3 test results +./scripts/testing/cleanup-tests.sh --keep 3 + +# Force without prompts +./scripts/testing/cleanup-tests.sh --force +#+end_src + +* Directory Structure + +#+begin_example +archsetup/ +├── scripts/ +│ └── testing/ +│ ├── README.org # This file +│ ├── setup-testing-env.sh # Setup infrastructure +│ ├── create-base-vm.sh # Create base VM +│ ├── run-test.sh # Run tests +│ ├── debug-vm.sh # Interactive debugging +│ ├── cleanup-tests.sh # Clean up +│ ├── finalize-base-vm.sh # Finalize base (generated) +│ ├── archinstall-config.json # Archinstall config +│ └── lib/ +│ ├── logging.sh # Logging utilities +│ └── vm-utils.sh # VM management +├── vm-images/ # VM disk images (gitignored) +│ ├── archsetup-base.qcow2 # Golden image +│ ├── arch-latest.iso # Arch ISO +│ └── archsetup-test-*.qcow2 # Test VMs +├── test-results/ # Test results (gitignored) +│ ├── TIMESTAMP/ +│ │ ├── test.log +│ │ ├── test-report.txt +│ │ └── archsetup-*.log +│ └── latest -> TIMESTAMP/ # Symlink to latest +└── docs/ + └── testing-strategy.org # Complete strategy doc +#+end_example + +* Configuration + +** VM Specifications + +All test VMs use: +- *CPUs:* 2 vCPUs +- *RAM:* 4GB (matches archsetup tmpfs build directory) +- *Disk:* 50GB (thin provisioned qcow2) +- *Network:* NAT via libvirt default network +- *Boot:* UEFI (systemd-boot bootloader) +- *Display:* Serial console + VNC + +Set environment variables to customize: +#+begin_src bash +VM_CPUS=4 VM_RAM=8192 ./scripts/testing/run-test.sh +#+end_src + +** Base VM Specifications + +The base VM contains a minimal Arch installation: +- Base system packages +- Linux kernel and firmware +- OpenSSH server (for automation) +- dhcpcd (for networking) +- git, vim, sudo (essentials) +- Root password: "archsetup" + +This matches the documented prerequisites for archsetup. + +* Validation Checks + +Each test run performs these validation checks: + +** Critical (Must Pass) +1. archsetup exits with code 0 +2. User 'cjennings' was created +3. Dotfiles are stowed (symlinks exist) +4. yay (AUR helper) is installed +5. DWM is built and installed + +** Additional (Future) +- All expected packages installed +- X11 can start +- systemd services enabled +- Firewall configured + +* Troubleshooting + +** VM fails to start + +Check if libvirtd is running: +#+begin_src bash +sudo systemctl status libvirtd +sudo systemctl start libvirtd +#+end_src + +** Cannot get VM IP address + +The VM may not have booted completely or networking failed: +#+begin_src bash +# Check VM status +virsh domstate VM_NAME + +# Connect to console to debug +virsh console VM_NAME + +# Check if dhcpcd is running in VM +systemctl status dhcpcd +#+end_src + +** SSH connection refused + +Wait longer for VM to boot, or check if sshd is enabled: +#+begin_src bash +virsh console VM_NAME +# Inside VM: +systemctl status sshd +systemctl start sshd +#+end_src + +** KVM not available + +Check if virtualization is enabled in BIOS and KVM modules loaded: +#+begin_src bash +# Check for /dev/kvm +ls -l /dev/kvm + +# Load KVM module (Intel) +sudo modprobe kvm-intel + +# Or for AMD +sudo modprobe kvm-amd + +# Verify +lsmod | grep kvm +#+end_src + +** Disk space issues + +Clean up old tests: +#+begin_src bash +./scripts/testing/cleanup-tests.sh --force +#+end_src + +Check disk usage: +#+begin_src bash +du -sh vm-images/ test-results/ +#+end_src + +** Base VM Installation Issues + +*** Firewall blocking VM network + +If the VM cannot reach the internet (100% packet loss when pinging), check the host firewall: + +#+begin_src bash +# Check UFW status on host +sudo ufw status + +# Check libvirt NAT rules on host +sudo iptables -t nat -L -n -v | grep -i libvirt +#+end_src + +*Solution:* Temporarily disable UFW during base VM creation: +#+begin_src bash +sudo ufw disable +# Create base VM +sudo ufw enable +#+end_src + +Or add libvirt rules to UFW: +#+begin_src bash +sudo ufw allow in on virbr0 +sudo ufw allow out on virbr0 +#+end_src + +*** VM network not working after boot + +If =dhcpcd= isn't running or network isn't configured: + +#+begin_src bash +# In the VM - restart network from scratch +killall dhcpcd + +# Bring interface down and up (replace enp1s0 with your interface) +ip link set enp1s0 down +ip link set enp1s0 up + +# Start dhcpcd +dhcpcd enp1s0 + +# Wait and verify +sleep 3 +ip addr show enp1s0 +ip route + +# Test connectivity +ping -c 3 192.168.122.1 # Gateway +ping -c 3 8.8.8.8 # Google DNS +#+end_src + +*** DNS not working (127.0.0.53 in resolv.conf) + +The Live ISO uses systemd-resolved stub resolver which may not work: + +#+begin_src bash +# In the VM - set real DNS servers +echo "nameserver 8.8.8.8" > /etc/resolv.conf +echo "nameserver 1.1.1.1" >> /etc/resolv.conf + +# Test +ping -c 3 archlinux.org +#+end_src + +*** Cannot paste into virt-viewer terminal + +Clipboard integration doesn't work well with virt-viewer. Use HTTP server instead: + +#+begin_src bash +# On host - serve the installation script +cd vm-images +python -m http.server 8000 + +# In VM - download and run +curl http://192.168.122.1:8000/auto-install.sh | bash + +# Or download first to review +curl http://192.168.122.1:8000/auto-install.sh -o install.sh +cat install.sh +bash install.sh +#+end_src + +*** Partitions "in use" error + +If re-running installation after a failed attempt: + +#+begin_src bash +# In the VM - unmount and wipe partitions +mount | grep vda +umount /mnt/boot 2>/dev/null +umount /mnt 2>/dev/null +umount -l /mnt # Lazy unmount if still busy + +# Wipe partition table completely +wipefs -a /dev/vda + +# Run install script again +bash install.sh +#+end_src + +*** Alternative: Use archinstall + +Instead of the auto-install.sh script, you can use Arch's built-in installer: + +#+begin_src bash +# In the VM +archinstall +#+end_src + +*Recommended settings:* +- Disk: =/dev/vda= +- Filesystem: =ext4= +- Bootloader: =systemd-boot= +- Hostname: =archsetup-test= +- Root password: =archsetup= +- Profile: =minimal= +- Additional packages: =openssh dhcpcd git vim sudo= +- Network: =NetworkManager= or =systemd-networkd= + +*After installation, before rebooting:* +#+begin_src bash +# Chroot into new system +arch-chroot /mnt + +# Enable services +systemctl enable sshd +systemctl enable dhcpcd # or NetworkManager + +# Allow root SSH login +sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config + +# Exit and poweroff +exit +poweroff +#+end_src + +* Future Enhancements + +** Planned Improvements +- [ ] Fully automated base VM creation (expect-based console automation) +- [ ] Parallel test execution (multiple VMs) +- [ ] Screenshot capture of X11 desktop +- [ ] CI/CD integration (GitHub Actions / GitLab CI) +- [ ] Performance benchmarking over time +- [ ] Cloud-init based base image (faster provisioning) + +** Test Scenarios +- [ ] Idempotency test (run archsetup twice) +- [ ] Network failure recovery +- [ ] Offline installation (local package cache) +- [ ] Different hardware profiles (CPU/RAM variations) + +* References + +- [[file:../../docs/testing-strategy.org][Testing Strategy Document]] +- [[https://wiki.archlinux.org/title/Libvirt][Arch Wiki: libvirt]] +- [[https://wiki.archlinux.org/title/QEMU][Arch Wiki: QEMU]] +- [[file:../../archsetup][Main archsetup script]] +- [[file:../../TODO.org][Project TODO]] diff --git a/scripts/testing/archinstall-config.json b/scripts/testing/archinstall-config.json new file mode 100644 index 0000000..a55e2a1 --- /dev/null +++ b/scripts/testing/archinstall-config.json @@ -0,0 +1,117 @@ +{ + "!users": { + "0": { + "!password": "archsetup", + "username": "root", + "sudo": false + } + }, + "archinstall-language": "English", + "audio_config": null, + "bootloader": "systemd-bootctl", + "config_version": "2.8.0", + "debug": false, + "disk_config": { + "config_type": "default_layout", + "device_modifications": [ + { + "device": "/dev/vda", + "partitions": [ + { + "btrfs": [], + "flags": [ + "Boot" + ], + "fs_type": "fat32", + "length": { + "sector_size": null, + "total_size": null, + "unit": "MiB", + "value": 512 + }, + "mount_options": [], + "mountpoint": "/boot", + "obj_id": "boot_partition", + "start": { + "sector_size": null, + "total_size": null, + "unit": "MiB", + "value": 1 + }, + "status": "create", + "type": "primary" + }, + { + "btrfs": [], + "flags": [], + "fs_type": "ext4", + "length": { + "sector_size": null, + "total_size": null, + "unit": "MiB", + "value": 100 + }, + "mount_options": [], + "mountpoint": "/", + "obj_id": "root_partition", + "start": { + "sector_size": null, + "total_size": null, + "unit": "MiB", + "value": 513 + }, + "status": "create", + "type": "primary" + } + ], + "wipe": true + } + ] + }, + "disk_encryption": null, + "hostname": "archsetup-test", + "kernels": [ + "linux" + ], + "locale_config": { + "kb_layout": "us", + "sys_enc": "UTF-8", + "sys_lang": "en_US" + }, + "mirror_config": { + "custom_mirrors": [], + "mirror_regions": { + "United States": [ + "https://mirror.rackspace.com/archlinux/$repo/os/$arch", + "https://mirror.leaseweb.com/archlinux/$repo/os/$arch" + ] + } + }, + "network_config": { + "type": "nm" + }, + "no_pkg_lookups": false, + "ntp": true, + "offline": false, + "packages": [ + "openssh", + "dhcpcd", + "git", + "vim" + ], + "parallel downloads": 5, + "profile_config": { + "gfx_driver": "All open-source", + "greeter": null, + "profile": { + "custom_settings": {}, + "details": [], + "main": "Minimal" + } + }, + "script": "guided", + "silent": false, + "swap": false, + "timezone": "America/Chicago", + "version": "2.8.0" +} diff --git a/scripts/testing/cleanup-tests.sh b/scripts/testing/cleanup-tests.sh new file mode 100755 index 0000000..e4289a7 --- /dev/null +++ b/scripts/testing/cleanup-tests.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# Clean up old test VMs and artifacts +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 + +set -e + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source utilities +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" + +# Parse arguments +KEEP_LAST=5 +FORCE=false + +while [[ $# -gt 0 ]]; do + case $1 in + --keep) + KEEP_LAST="$2" + shift 2 + ;; + --force) + FORCE=true + shift + ;; + *) + echo "Usage: $0 [--keep N] [--force]" + echo " --keep N Keep last N test results (default: 5)" + echo " --force Skip confirmation prompts" + exit 1 + ;; + esac +done + +# Initialize logging +LOGFILE="/tmp/cleanup-tests-$(date +'%Y%m%d-%H%M%S').log" +init_logging "$LOGFILE" + +section "Cleaning Up Test Artifacts" + +# Find all test VMs +step "Finding test VMs" +TEST_VMS=$(virsh --connect qemu:///system list --all | grep "archsetup-test-" | awk '{print $2}' || true) + +if [ -z "$TEST_VMS" ]; then + info "No test VMs found" +else + VM_COUNT=$(echo "$TEST_VMS" | wc -l) + info "Found $VM_COUNT test VM(s)" + + if ! $FORCE; then + echo "" + echo "$TEST_VMS" + echo "" + read -p "Destroy these VMs? [y/N] " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + info "Skipping VM cleanup" + else + for vm in $TEST_VMS; do + step "Destroying VM: $vm" + if vm_is_running "$vm"; then + virsh --connect qemu:///system destroy "$vm" >> "$LOGFILE" 2>&1 + fi + virsh --connect qemu:///system undefine "$vm" --nvram >> "$LOGFILE" 2>&1 || true + success "VM destroyed: $vm" + done + fi + else + for vm in $TEST_VMS; do + step "Destroying VM: $vm" + if vm_is_running "$vm"; then + virsh --connect qemu:///system destroy "$vm" >> "$LOGFILE" 2>&1 + fi + virsh --connect qemu:///system undefine "$vm" --nvram >> "$LOGFILE" 2>&1 || true + success "VM destroyed: $vm" + done + fi +fi + +# Clean up test disk images +section "Cleaning Up Disk Images" + +step "Finding test disk images" +if [ -d "$PROJECT_ROOT/vm-images" ]; then + TEST_DISKS=$(find "$PROJECT_ROOT/vm-images" -name "archsetup-test-*.qcow2" 2>/dev/null || true) + + if [ -z "$TEST_DISKS" ]; then + info "No test disk images found" + else + DISK_COUNT=$(echo "$TEST_DISKS" | wc -l) + DISK_SIZE=$(du -ch $TEST_DISKS | tail -1 | awk '{print $1}') + info "Found $DISK_COUNT test disk image(s) totaling $DISK_SIZE" + + if ! $FORCE; then + echo "" + echo "$TEST_DISKS" + echo "" + read -p "Delete these disk images? [y/N] " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + info "Skipping disk cleanup" + else + echo "$TEST_DISKS" | while read disk; do + rm -f "$disk" + done + success "Test disk images deleted" + fi + else + echo "$TEST_DISKS" | while read disk; do + rm -f "$disk" + done + success "Test disk images deleted" + fi + fi +fi + +# Clean up old test results +section "Cleaning Up Test Results" + +if [ ! -d "$PROJECT_ROOT/test-results" ]; then + info "No test results directory" +else + step "Finding test result directories" + TEST_RESULTS=$(find "$PROJECT_ROOT/test-results" -maxdepth 1 -type d -name "20*" 2>/dev/null | sort -r || true) + + if [ -z "$TEST_RESULTS" ]; then + info "No test results found" + else + RESULT_COUNT=$(echo "$TEST_RESULTS" | wc -l) + info "Found $RESULT_COUNT test result directory(ies)" + + if [ $RESULT_COUNT -le $KEEP_LAST ]; then + info "Keeping all results (count <= $KEEP_LAST)" + else + TO_DELETE=$(echo "$TEST_RESULTS" | tail -n +$((KEEP_LAST + 1))) + DELETE_COUNT=$(echo "$TO_DELETE" | wc -l) + info "Keeping last $KEEP_LAST, deleting $DELETE_COUNT old result(s)" + + if ! $FORCE; then + echo "" + echo "Will delete:" + echo "$TO_DELETE" + echo "" + read -p "Delete these test results? [y/N] " -n 1 -r + echo "" + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + info "Skipping results cleanup" + else + echo "$TO_DELETE" | while read dir; do + rm -rf "$dir" + done + success "Old test results deleted" + fi + else + echo "$TO_DELETE" | while read dir; do + rm -rf "$dir" + done + success "Old test results deleted" + fi + fi + fi +fi + +section "Cleanup Complete" + +info "Log file: $LOGFILE" diff --git a/scripts/testing/create-base-vm.sh b/scripts/testing/create-base-vm.sh new file mode 100755 index 0000000..03409fe --- /dev/null +++ b/scripts/testing/create-base-vm.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Create base VM for archsetup testing - Manual Installation +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# This script creates a VM booted from Arch ISO, then waits for you to +# manually install Arch using archinstall. + +set -e + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source utilities +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" + +# Configuration +VM_NAME="archsetup-base" +VM_CPUS="${VM_CPUS:-4}" +VM_RAM="${VM_RAM:-8192}" # MB +VM_DISK="${VM_DISK:-50}" # GB +VM_IMAGES_DIR="$PROJECT_ROOT/vm-images" +ISO_URL="https://mirrors.kernel.org/archlinux/iso/latest/archlinux-x86_64.iso" +ISO_PATH="$VM_IMAGES_DIR/arch-latest.iso" +DISK_PATH="$VM_IMAGES_DIR/archsetup-base.qcow2" + +# Initialize logging +LOGFILE="$PROJECT_ROOT/test-results/create-base-vm-$(date +'%Y%m%d-%H%M%S').log" +init_logging "$LOGFILE" + +section "Creating Base VM for ArchSetup Testing" + +# Verify prerequisites +step "Checking prerequisites" +check_libvirt || fatal "libvirt not running" +check_libvirt_group || fatal "User not in libvirt group" +check_kvm || fatal "KVM not available" +success "Prerequisites satisfied" + +# Create vm-images directory +mkdir -p "$VM_IMAGES_DIR" + +# Download Arch ISO if needed +section "Preparing Arch Linux ISO" + +if [ -f "$ISO_PATH" ]; then + info "Arch ISO exists: $ISO_PATH" + + # Check if ISO is older than 30 days + if [ $(find "$ISO_PATH" -mtime +30 | wc -l) -gt 0 ]; then + warn "ISO is older than 30 days" + info "Downloading latest version..." + rm -f "$ISO_PATH" + else + success "Using existing ISO" + fi +fi + +if [ ! -f "$ISO_PATH" ]; then + step "Downloading latest Arch ISO" + info "URL: $ISO_URL" + info "This may take several minutes..." + + if wget --progress=dot:giga -O "$ISO_PATH" "$ISO_URL" 2>&1 | tee -a "$LOGFILE"; then + success "ISO downloaded" + else + fatal "ISO download failed" + fi +fi + +# Remove existing VM and disk +if vm_exists "$VM_NAME"; then + warn "VM $VM_NAME already exists - destroying it" + if vm_is_running "$VM_NAME"; then + virsh destroy "$VM_NAME" >> "$LOGFILE" 2>&1 + fi + virsh undefine "$VM_NAME" --nvram >> "$LOGFILE" 2>&1 || true +fi + +[ -f "$DISK_PATH" ] && rm -f "$DISK_PATH" + +# Create and start VM +section "Creating and Starting VM" + +info "Creating VM: $VM_NAME" +info " CPUs: $VM_CPUS | RAM: ${VM_RAM}MB | Disk: ${VM_DISK}GB" + +virt-install \ + --connect qemu:///system \ + --name "$VM_NAME" \ + --memory "$VM_RAM" \ + --vcpus "$VM_CPUS" \ + --disk path="$DISK_PATH",size="$VM_DISK",format=qcow2,bus=virtio \ + --cdrom "$ISO_PATH" \ + --os-variant archlinux \ + --network network=default,model=virtio \ + --graphics vnc,listen=127.0.0.1 \ + --console pty,target.type=serial \ + --boot uefi \ + --noreboot \ + --check path_in_use=off \ + --filesystem type=mount,mode=mapped,source="$PROJECT_ROOT/scripts",target=host-scripts \ + >> "$LOGFILE" 2>&1 & + +VIRT_INSTALL_PID=$! + +progress "Waiting for VM to boot from ISO" +sleep 30 + +# Check if VM started +if ! vm_is_running "$VM_NAME"; then + wait $VIRT_INSTALL_PID + EXIT_CODE=$? + fatal "VM failed to start (exit code: $EXIT_CODE)" +fi + +success "VM started successfully" + +# Display manual installation instructions +section "Manual Installation Required" + +cat << 'EOF' + +[i] +[i] Base VM is running from Arch ISO +[i] +[i] NEXT STEPS - Complete installation manually: +[i] +[i] 1. Open virt-viewer (should already be open): +[i] virt-viewer --connect qemu:///system archsetup-base +[i] +[i] 2. Login as 'root' (no password) +[i] +[i] 3. Run: archinstall +[i] +[i] 4. Configure with these settings: +[i] - Hostname: archsetup-test +[i] - Root password: archsetup +[i] - Profile: minimal +[i] - Network: dhcpcd (or NetworkManager) +[i] - Additional packages: openssh git vim sudo iperf3 mtr traceroute bind net-tools sshfs +[i] - Enable: sshd, dhcpcd (or NetworkManager) +[i] +[i] 5. After archinstall completes: +[i] - Chroot into /mnt: arch-chroot /mnt +[i] - Edit /etc/ssh/sshd_config: +[i] sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config +[i] sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config +[i] - Set up shared folder mount (9p filesystem): +[i] mkdir -p /mnt/host-scripts +[i] echo 'host-scripts /mnt/host-scripts 9p trans=virtio,version=9p2000.L,rw 0 0' >> /etc/fstab +[i] - Exit chroot: exit +[i] - Poweroff: poweroff +[i] +[i] 6. After VM powers off, run: +[i] ./scripts/testing/finalize-base-vm.sh +[i] +[i] Log file: $LOGFILE +[i] + +EOF + +info "Waiting for VM to power off..." +info "(This script will exit when you manually power off the VM)" + +# Wait for virt-install to finish (VM powers off) +wait $VIRT_INSTALL_PID || true + +success "VM has powered off" +info "" +info "Next step: Run ./scripts/testing/finalize-base-vm.sh" diff --git a/scripts/testing/debug-vm.sh b/scripts/testing/debug-vm.sh new file mode 100755 index 0000000..a442850 --- /dev/null +++ b/scripts/testing/debug-vm.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Launch VM for interactive debugging +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 + +set -e + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source utilities +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" + +# Parse arguments +VM_DISK="" +USE_BASE=false + +if [ $# -eq 0 ]; then + USE_BASE=true +elif [ "$1" = "--base" ]; then + USE_BASE=true +elif [ -f "$1" ]; then + VM_DISK="$1" +else + echo "Usage: $0 [disk-image.qcow2 | --base]" + echo "" + echo "Options:" + echo " --base Use base VM (read-only, safe for testing)" + echo " disk-image.qcow2 Use existing test disk image" + echo " (no args) Clone base VM for debugging" + exit 1 +fi + +# Configuration +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +DEBUG_VM_NAME="archsetup-debug-$TIMESTAMP" +BASE_DISK="$PROJECT_ROOT/vm-images/archsetup-base.qcow2" +ROOT_PASSWORD="archsetup" + +# Initialize logging +LOGFILE="/tmp/debug-vm-$TIMESTAMP.log" +init_logging "$LOGFILE" + +section "Launching Debug VM" + +# Determine which disk to use +if $USE_BASE; then + if [ ! -f "$BASE_DISK" ]; then + fatal "Base disk not found: $BASE_DISK" + fi + VM_DISK="$BASE_DISK" + info "Using base VM (read-only snapshot mode)" +else + if [ -z "$VM_DISK" ]; then + # Clone base VM + VM_DISK="$PROJECT_ROOT/vm-images/$DEBUG_VM_NAME.qcow2" + step "Cloning base VM for debugging" + clone_disk "$BASE_DISK" "$VM_DISK" || fatal "Failed to clone base VM" + success "Debug disk created: $VM_DISK" + else + info "Using existing disk: $VM_DISK" + fi +fi + +# Create debug VM +step "Creating debug VM: $DEBUG_VM_NAME" +virt-install \ + --connect qemu:///system \ + --name "$DEBUG_VM_NAME" \ + --memory 4096 \ + --vcpus 2 \ + --disk path="$VM_DISK",format=qcow2,bus=virtio \ + --os-variant archlinux \ + --network network=default,model=virtio \ + --graphics vnc,listen=127.0.0.1 \ + --console pty,target.type=serial \ + --boot uefi \ + --import \ + --noautoconsole \ + >> "$LOGFILE" 2>&1 + +success "Debug VM created" + +# Wait for boot +step "Waiting for VM to boot..." +sleep 20 + +# Get VM IP +VM_IP=$(get_vm_ip "$DEBUG_VM_NAME" 2>/dev/null || true) + +# Display connection information +section "Debug VM Ready" + +info "" +info "VM Name: $DEBUG_VM_NAME" +if [ -n "$VM_IP" ]; then + info "IP Address: $VM_IP" +fi +info "Disk: $VM_DISK" +info "" +info "Connect via:" +info " Console: virsh console $DEBUG_VM_NAME" +if [ -n "$VM_IP" ]; then + info " SSH: ssh root@$VM_IP" +fi +info " VNC: virt-viewer $DEBUG_VM_NAME" +info "" +info "Root password: $ROOT_PASSWORD" +info "" +info "When done debugging:" +info " virsh destroy $DEBUG_VM_NAME" +info " virsh undefine $DEBUG_VM_NAME" +if [ ! "$VM_DISK" = "$BASE_DISK" ] && [ -z "$1" ]; then + info " rm $VM_DISK" +fi +info "" +info "Log file: $LOGFILE" +info "" + +# Offer to connect to console +read -p "Connect to console now? [Y/n] " -n 1 -r +echo "" +if [[ $REPLY =~ ^[Nn]$ ]]; then + info "VM is running in background" + info "Connect later with: virsh console $DEBUG_VM_NAME" +else + info "Connecting to console..." + info "Press Ctrl+] to disconnect from console" + sleep 2 + virsh console "$DEBUG_VM_NAME" +fi diff --git a/scripts/testing/finalize-base-vm.sh b/scripts/testing/finalize-base-vm.sh new file mode 100755 index 0000000..225ffae --- /dev/null +++ b/scripts/testing/finalize-base-vm.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Finalize base VM after installation +VM_NAME="archsetup-base" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +BASE_DISK="$PROJECT_ROOT/vm-images/archsetup-base.qcow2" + +echo "[i] Removing ISO from VM..." +virsh --connect qemu:///system change-media $VM_NAME sda --eject 2>/dev/null || true +virsh --connect qemu:///system change-media $VM_NAME hda --eject 2>/dev/null || true +echo "[✓] ISO removed" + +echo "[i] Fixing base disk permissions..." +sudo chown $USER:$USER "$BASE_DISK" +sudo chmod 644 "$BASE_DISK" +echo "[✓] Permissions fixed" + +echo "[i] Starting VM from installed system..." +virsh --connect qemu:///system start $VM_NAME +echo "[i] Waiting for boot..." +sleep 30 +IP=$(virsh --connect qemu:///system domifaddr $VM_NAME 2>/dev/null | grep -oP '(\d+\.){3}\d+' | head -1) +echo "[✓] Base VM is ready!" +echo "" +echo "Connect via:" +echo " Console: virsh console $VM_NAME" +echo " SSH: ssh root@$IP" +echo " Password: archsetup" +echo "" +echo "To create a test clone:" +echo " ./scripts/testing/run-test.sh" diff --git a/scripts/testing/lib/finalize-base-vm.sh b/scripts/testing/lib/finalize-base-vm.sh new file mode 100755 index 0000000..e3913ea --- /dev/null +++ b/scripts/testing/lib/finalize-base-vm.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Finalize base VM after installation +VM_NAME="archsetup-base" +echo "[i] Removing ISO from VM..." +virsh change-media $VM_NAME sda --eject 2>/dev/null || true +virsh change-media $VM_NAME hda --eject 2>/dev/null || true +echo "[✓] ISO removed" +echo "[i] Starting VM from installed system..." +virsh start $VM_NAME +echo "[i] Waiting for boot..." +sleep 30 +IP=$(virsh domifaddr $VM_NAME 2>/dev/null | grep -oP '(\d+\.){3}\d+' | head -1) +echo "[✓] Base VM is ready!" +echo "" +echo "Connect via:" +echo " Console: virsh console $VM_NAME" +echo " SSH: ssh root@$IP" +echo " Password: archsetup" +echo "" +echo "To create a test clone:" +echo " ./scripts/testing/run-test.sh" diff --git a/scripts/testing/lib/logging.sh b/scripts/testing/lib/logging.sh new file mode 100755 index 0000000..eda9eb1 --- /dev/null +++ b/scripts/testing/lib/logging.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# Logging utilities for archsetup testing +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 + +# Global log file (set by calling script) +LOGFILE="${LOGFILE:-/tmp/archsetup-test.log}" + +# Initialize logging +init_logging() { + local logfile="$1" + LOGFILE="$logfile" + + # Create log directory if it doesn't exist + mkdir -p "$(dirname "$LOGFILE")" + + # Initialize log file + echo "=== Test Log Started: $(date +'%Y-%m-%d %H:%M:%S') ===" > "$LOGFILE" + echo "" >> "$LOGFILE" +} + +# Log message (to file and optionally stdout) +log() { + local message="$1" + local timestamp + timestamp=$(date +'%Y-%m-%d %H:%M:%S') + echo "[$timestamp] $message" >> "$LOGFILE" +} + +# Info message +info() { + local message="$1" + echo "[i] $message" + log "INFO: $message" +} + +# Success message +success() { + local message="$1" + echo "[✓] $message" + log "SUCCESS: $message" +} + +# Warning message +warn() { + local message="$1" + echo "[!] $message" + log "WARNING: $message" +} + +# Error message +error() { + local message="$1" + echo "[✗] $message" >&2 + log "ERROR: $message" +} + +# Fatal error (exits script) +fatal() { + local message="$1" + local exit_code="${2:-1}" + echo "[✗] FATAL: $message" >&2 + log "FATAL: $message (exit code: $exit_code)" + exit "$exit_code" +} + +# Section header +section() { + local title="$1" + echo "" + echo "=== $title ===" + log "=== $title ===" +} + +# Step message +step() { + local message="$1" + echo " -> $message" + log " STEP: $message" +} + +# Progress indicator (for long-running operations) +progress() { + local message="$1" + echo " ... $message" + log " PROGRESS: $message" +} + +# Clear progress line and show completion +complete() { + local message="$1" + echo " [✓] $message" + log " COMPLETE: $message" +} + +# Show command being executed (useful for debugging) +show_cmd() { + local cmd="$1" + echo "$ $cmd" + log "CMD: $cmd" +} + +# Separator line +separator() { + echo "----------------------------------------" +} + +# Summary statistics +summary() { + local passed="$1" + local failed="$2" + local total=$((passed + failed)) + + echo "" + separator + section "Test Summary" + echo " Total: $total" + echo " Passed: $passed" + echo " Failed: $failed" + separator + echo "" + + log "=== Test Summary ===" + log "Total: $total, Passed: $passed, Failed: $failed" +} + +# Timer utilities +declare -A TIMERS + +start_timer() { + local name="${1:-default}" + TIMERS[$name]=$(date +%s) + log "TIMER START: $name" +} + +stop_timer() { + local name="${1:-default}" + local start=${TIMERS[$name]} + local end=$(date +%s) + local duration=$((end - start)) + local mins=$((duration / 60)) + local secs=$((duration % 60)) + + if [ $mins -gt 0 ]; then + echo " Time: ${mins}m ${secs}s" + log "TIMER STOP: $name (${mins}m ${secs}s)" + else + echo " Time: ${secs}s" + log "TIMER STOP: $name (${secs}s)" + fi +} diff --git a/scripts/testing/lib/network-diagnostics.sh b/scripts/testing/lib/network-diagnostics.sh new file mode 100644 index 0000000..3f9735b --- /dev/null +++ b/scripts/testing/lib/network-diagnostics.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Network diagnostics for VM testing +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 + +# Note: logging.sh should already be sourced by the calling script + +# Run quick network diagnostics +# Args: $1 = VM IP address or hostname +run_network_diagnostics() { + local vm_host="$1" + + section "Pre-flight Network Diagnostics" + + # Test 1: Basic connectivity + step "Testing internet connectivity" + if sshpass -p 'archsetup' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$vm_host "ping -c 3 8.8.8.8 >/dev/null 2>&1"; then + success "Internet connectivity OK" + else + error "No internet connectivity" + return 1 + fi + + # Test 2: DNS resolution + step "Testing DNS resolution" + if sshpass -p 'archsetup' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$vm_host "nslookup archlinux.org >/dev/null 2>&1"; then + success "DNS resolution OK" + else + error "DNS resolution failed" + return 1 + fi + + # Test 3: Arch mirror accessibility + step "Testing Arch mirror access" + if sshpass -p 'archsetup' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$vm_host "curl -s -I https://mirrors.kernel.org/archlinux/ | head -1 | grep -qE '(200|301)'"; then + success "Arch mirrors accessible" + else + error "Cannot reach Arch mirrors" + return 1 + fi + + # Test 4: AUR accessibility + step "Testing AUR access" + if sshpass -p 'archsetup' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$vm_host "curl -s -I https://aur.archlinux.org/ | head -1 | grep -qE '(200|405)'"; then + success "AUR accessible" + else + error "Cannot reach AUR" + return 1 + fi + + # Show network info + info "Network configuration:" + sshpass -p 'archsetup' ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$vm_host \ + "ip addr show | grep 'inet ' | grep -v '127.0.0.1'" 2>/dev/null | while read line; do + info " $line" + done + + success "Network diagnostics complete" + return 0 +} diff --git a/scripts/testing/lib/validation.sh b/scripts/testing/lib/validation.sh new file mode 100644 index 0000000..c0a5d43 --- /dev/null +++ b/scripts/testing/lib/validation.sh @@ -0,0 +1,1080 @@ +#!/bin/bash +# Validation utilities for archsetup testing +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# This module provides comprehensive validation checks for archsetup installations. +# It captures pre-install state, runs post-install validations, and attributes +# issues to either archsetup or the base install (archzfs/vanilla Arch). + +# Validation counters +VALIDATION_PASSED=0 +VALIDATION_FAILED=0 +VALIDATION_WARNINGS=0 + +# Arrays to track issues +declare -a ARCHSETUP_ISSUES +declare -a BASE_INSTALL_ISSUES +declare -a UNKNOWN_ISSUES + +# SSH helper (uses globals: VM_IP, ROOT_PASSWORD) +ssh_cmd() { + sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 "root@$VM_IP" "$@" 2>/dev/null +} + +# Validation result helpers +validation_pass() { + local test_name="$1" + success "$test_name" + ((VALIDATION_PASSED++)) || true +} + +validation_fail() { + local test_name="$1" + local details="${2:-}" + error "$test_name" + [ -n "$details" ] && info " Details: $details" + ((VALIDATION_FAILED++)) || true +} + +validation_warn() { + local test_name="$1" + local details="${2:-}" + warn "$test_name" + [ -n "$details" ] && info " Details: $details" + ((VALIDATION_WARNINGS++)) || true +} + +# Attribute an issue to archsetup or base install +attribute_issue() { + local issue="$1" + local source="$2" # "archsetup", "base", or "unknown" + + case "$source" in + archsetup) + ARCHSETUP_ISSUES+=("$issue") + ;; + base) + BASE_INSTALL_ISSUES+=("$issue") + ;; + *) + UNKNOWN_ISSUES+=("$issue") + ;; + esac +} + +#============================================================================= +# PRE-INSTALL LOG CAPTURE +#============================================================================= + +capture_pre_install_state() { + local output_dir="$1" + + section "Capturing Pre-Install State" + mkdir -p "$output_dir/pre-install" + + step "Capturing system logs before archsetup" + + # Capture journal + ssh_cmd "journalctl -b --no-pager" > "$output_dir/pre-install/journal.log" 2>&1 || true + + # Capture dmesg + ssh_cmd "dmesg" > "$output_dir/pre-install/dmesg.log" 2>&1 || true + + # Capture package list + ssh_cmd "pacman -Q" > "$output_dir/pre-install/packages.txt" 2>&1 || true + + # Capture service status + ssh_cmd "systemctl list-units --type=service --all" > "$output_dir/pre-install/services.txt" 2>&1 || true + + # Capture failed services + ssh_cmd "systemctl --failed" > "$output_dir/pre-install/failed-services.txt" 2>&1 || true + + # Capture existing errors in logs + ssh_cmd "journalctl -b -p err --no-pager" > "$output_dir/pre-install/errors.log" 2>&1 || true + + # Count pre-existing errors + PRE_INSTALL_ERROR_COUNT=$(wc -l < "$output_dir/pre-install/errors.log" 2>/dev/null || echo 0) + + success "Pre-install state captured ($PRE_INSTALL_ERROR_COUNT pre-existing error lines)" +} + +#============================================================================= +# POST-INSTALL LOG CAPTURE +#============================================================================= + +capture_post_install_state() { + local output_dir="$1" + + section "Capturing Post-Install State" + mkdir -p "$output_dir/post-install" + + step "Capturing system logs after archsetup" + + # Capture journal + ssh_cmd "journalctl -b --no-pager" > "$output_dir/post-install/journal.log" 2>&1 || true + + # Capture dmesg + ssh_cmd "dmesg" > "$output_dir/post-install/dmesg.log" 2>&1 || true + + # Capture package list + ssh_cmd "pacman -Q" > "$output_dir/post-install/packages.txt" 2>&1 || true + + # Capture service status + ssh_cmd "systemctl list-units --type=service --all" > "$output_dir/post-install/services.txt" 2>&1 || true + + # Capture failed services + ssh_cmd "systemctl --failed" > "$output_dir/post-install/failed-services.txt" 2>&1 || true + + # Capture all errors + ssh_cmd "journalctl -b -p err --no-pager" > "$output_dir/post-install/errors.log" 2>&1 || true + + # Capture archsetup log + ssh_cmd "cat /var/log/archsetup-*.log 2>/dev/null" > "$output_dir/post-install/archsetup.log" 2>&1 || true + + success "Post-install state captured" +} + +#============================================================================= +# LOG DIFF ANALYSIS +#============================================================================= + +analyze_log_diff() { + local output_dir="$1" + + section "Analyzing Log Differences" + mkdir -p "$output_dir/analysis" + + step "Comparing pre and post install errors" + + # Find new errors (in post but not in pre) + if [ -f "$output_dir/pre-install/errors.log" ] && [ -f "$output_dir/post-install/errors.log" ]; then + comm -13 <(sort "$output_dir/pre-install/errors.log") <(sort "$output_dir/post-install/errors.log") \ + > "$output_dir/analysis/new-errors.log" 2>/dev/null || true + + NEW_ERROR_COUNT=$(wc -l < "$output_dir/analysis/new-errors.log" 2>/dev/null || echo 0) + + if [ "$NEW_ERROR_COUNT" -gt 0 ]; then + warn "Found $NEW_ERROR_COUNT new error lines after archsetup" + # Categorize errors + categorize_errors "$output_dir/analysis/new-errors.log" "$output_dir/analysis" + else + success "No new errors introduced by archsetup" + fi + fi + + step "Checking for new failed services" + + # Compare failed services + if [ -f "$output_dir/pre-install/failed-services.txt" ] && [ -f "$output_dir/post-install/failed-services.txt" ]; then + local pre_failed post_failed + pre_failed=$(grep -c "failed" "$output_dir/pre-install/failed-services.txt" 2>/dev/null | tr -d '[:space:]') + post_failed=$(grep -c "failed" "$output_dir/post-install/failed-services.txt" 2>/dev/null | tr -d '[:space:]') + # Default to 0 if empty + pre_failed=${pre_failed:-0} + post_failed=${post_failed:-0} + + if [ "$post_failed" -gt "$pre_failed" ]; then + warn "New failed services detected (before: $pre_failed, after: $post_failed)" + diff "$output_dir/pre-install/failed-services.txt" "$output_dir/post-install/failed-services.txt" \ + > "$output_dir/analysis/failed-services-diff.txt" 2>/dev/null || true + else + success "No new service failures" + fi + fi + + step "Counting new packages installed" + + if [ -f "$output_dir/pre-install/packages.txt" ] && [ -f "$output_dir/post-install/packages.txt" ]; then + comm -13 <(sort "$output_dir/pre-install/packages.txt") <(sort "$output_dir/post-install/packages.txt") \ + > "$output_dir/analysis/new-packages.txt" 2>/dev/null || true + + local new_pkg_count=$(wc -l < "$output_dir/analysis/new-packages.txt" 2>/dev/null || echo 0) + info "Installed $new_pkg_count new packages" + fi +} + +categorize_errors() { + local error_log="$1" + local output_dir="$2" + + # Known benign errors/warnings to ignore + local -a BENIGN_PATTERNS=( + "SPL:.*module verification failed" + "ZFS:.*module verification failed" + "tainting kernel" + "RAS:.*Correctable Errors" + "ACPI.*AE_NOT_FOUND" + "firmware.*regulatory" + "Invalid user name.*in service file" # dbus-broker timing during package install + ) + + # Patterns that indicate archsetup issues + local -a ARCHSETUP_PATTERNS=( + "archsetup" + "stow" + "yay" + "makepkg" + "pacman.*error" + ) + + # Filter and categorize + while IFS= read -r line; do + local is_benign=false + local is_archsetup=false + + # Check if benign + for pattern in "${BENIGN_PATTERNS[@]}"; do + if echo "$line" | grep -qiE "$pattern"; then + is_benign=true + break + fi + done + + if $is_benign; then + echo "$line" >> "$output_dir/benign-errors.log" + continue + fi + + # Check if archsetup-related + for pattern in "${ARCHSETUP_PATTERNS[@]}"; do + if echo "$line" | grep -qiE "$pattern"; then + is_archsetup=true + break + fi + done + + if $is_archsetup; then + echo "$line" >> "$output_dir/archsetup-errors.log" + attribute_issue "$line" "archsetup" + else + echo "$line" >> "$output_dir/base-install-errors.log" + attribute_issue "$line" "base" + fi + done < "$error_log" +} + +#============================================================================= +# VALIDATION CHECKS +#============================================================================= + +run_all_validations() { + section "Running Validation Checks" + + # User & Authentication + validate_user_created + validate_user_shell + validate_user_groups + + # Dotfiles + validate_dotfiles + + # Package Managers + validate_yay_installed + validate_pacman_working + + # Window Manager (detects DWM or Hyprland automatically) + validate_window_manager + + # Essential Services + validate_firewall + validate_dns_config + validate_avahi + validate_fail2ban + validate_networkmanager + + # Developer Tools + validate_emacs + validate_git_config + validate_dev_tools + + # System Configuration + validate_zfs_config + validate_boot_config + validate_autologin_config + validate_gnome_keyring_setup + + # Boot & Initramfs (critical for ZFS systems) + validate_terminus_font + validate_mkinitcpio_hooks + validate_initramfs_consolefont + validate_nvme_module + + # Archsetup Specific + validate_archsetup_log + validate_state_markers +} + +#----------------------------------------------------------------------------- +# User & Authentication Validations +#----------------------------------------------------------------------------- + +validate_user_created() { + step "Checking if user 'cjennings' exists" + if ssh_cmd "id cjennings" &>> "$LOGFILE"; then + validation_pass "User cjennings exists" + else + validation_fail "User cjennings not found" + attribute_issue "User cjennings not created" "archsetup" + fi +} + +validate_user_shell() { + step "Checking if ZSH is default shell" + local shell=$(ssh_cmd "getent passwd cjennings | cut -d: -f7") + if [ "$shell" = "/bin/zsh" ] || [ "$shell" = "/usr/bin/zsh" ]; then + validation_pass "ZSH is default shell" + else + validation_fail "ZSH not default shell (got: $shell)" + attribute_issue "ZSH not set as default shell" "archsetup" + fi +} + +validate_user_groups() { + step "Checking user group memberships" + # Groups added by archsetup: + # - wheel (useradd -G wheel) + # - sys,adm,network,scanner,power,uucp,audio,lp,rfkill,video,storage,optical,users (usermod -aG) + # - docker (gpasswd -a, added later in developer_workstation) + local expected_groups="wheel sys adm network scanner power uucp audio lp rfkill video storage optical users docker" + local missing_groups="" + + for group in $expected_groups; do + if ! ssh_cmd "groups cjennings" | grep -q "\b$group\b"; then + missing_groups="$missing_groups $group" + fi + done + + if [ -z "$missing_groups" ]; then + validation_pass "User in all expected groups (15 groups)" + else + validation_fail "User missing groups:$missing_groups" + attribute_issue "User missing groups:$missing_groups" "archsetup" + fi +} + +#----------------------------------------------------------------------------- +# Dotfiles Validations +#----------------------------------------------------------------------------- + +validate_dotfiles() { + step "Checking dotfiles setup" + + # 1. Check if .zshrc is a symlink + if ! ssh_cmd "test -L /home/cjennings/.zshrc"; then + validation_fail "Dotfiles not stowed (.zshrc is not a symlink)" + attribute_issue "Dotfiles stow failed" "archsetup" + return 1 + fi + + # 2. Check symlink points to correct location + local target=$(ssh_cmd "readlink /home/cjennings/.zshrc") + local expected_pattern="code/archsetup/dotfiles/system/.zshrc" + + if ! echo "$target" | grep -q "$expected_pattern"; then + validation_fail "Dotfiles symlink points to wrong location: $target" + attribute_issue "Dotfiles symlink incorrect: $target" "archsetup" + return 1 + fi + + # 3. Check the target file actually exists (not a broken symlink) + if ! ssh_cmd "test -f /home/cjennings/.zshrc"; then + validation_fail "Dotfiles symlink is broken (target doesn't exist)" + ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 + attribute_issue "Dotfiles symlink broken" "archsetup" + return 1 + fi + + # 4. Check user can actually read the file (not just root) + local result=$(ssh_cmd "sudo -u cjennings cat /home/cjennings/.zshrc > /dev/null 2>&1 && echo OK || echo FAIL") + if [ "$result" != "OK" ]; then + validation_fail "Dotfiles not readable by user (permission issue)" + ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 + attribute_issue "Dotfiles not readable by user" "archsetup" + return 1 + fi + + validation_pass "Dotfiles configured correctly (symlink to $target, readable by user)" +} + +#----------------------------------------------------------------------------- +# Package Manager Validations +#----------------------------------------------------------------------------- + +validate_yay_installed() { + step "Checking if yay (AUR helper) is installed and functional" + + # Check binary exists + if ! ssh_cmd "which yay" &>> "$LOGFILE"; then + validation_fail "yay not found" + attribute_issue "yay not installed" "archsetup" + return 1 + fi + + # Check yay can query packages (functional test) + if ssh_cmd "sudo -u cjennings yay -Qi yay" &>> "$LOGFILE"; then + validation_pass "yay is installed and functional" + else + validation_fail "yay binary exists but query failed" + attribute_issue "yay not functional" "archsetup" + fi +} + +validate_pacman_working() { + step "Checking if pacman is functional" + if ssh_cmd "pacman -Qi base" &>> "$LOGFILE"; then + validation_pass "pacman is functional" + else + validation_fail "pacman query failed" + attribute_issue "pacman not functional" "unknown" + fi +} + +#----------------------------------------------------------------------------- +# Window Manager Validations +#----------------------------------------------------------------------------- + +validate_suckless_tools() { + step "Checking suckless tools (dwm, st, dmenu, slock)" + local missing="" + + for tool in dwm st dmenu slock; do + if ! ssh_cmd "test -f /usr/local/bin/$tool"; then + missing="$missing $tool" + fi + done + + if [ -z "$missing" ]; then + validation_pass "All suckless tools installed (dwm, st, dmenu, slock)" + else + validation_fail "Missing suckless tools:$missing" + attribute_issue "Missing suckless tools:$missing" "archsetup" + fi +} + +validate_hyprland_tools() { + step "Checking Hyprland tools" + local missing="" + + # Check core Hyprland packages + for pkg in hyprland hypridle hyprlock waybar wofi swww grim slurp gammastep; do + if ! ssh_cmd "pacman -Q $pkg &>/dev/null"; then + missing="$missing $pkg" + fi + done + + # st should still be installed (via XWayland) + if ! ssh_cmd "test -f /usr/local/bin/st"; then + missing="$missing st" + fi + + if [ -z "$missing" ]; then + validation_pass "All Hyprland tools installed" + else + validation_fail "Missing Hyprland tools:$missing" + attribute_issue "Missing Hyprland tools:$missing" "archsetup" + fi +} + +validate_hyprland_config() { + step "Checking Hyprland configuration files" + local missing="" + + for config in ".config/hypr/hyprland.conf" ".config/hypr/hypridle.conf" \ + ".config/hypr/hyprlock.conf" ".config/waybar/config" \ + ".config/wofi/config" ".config/gammastep/config.ini"; do + if ! ssh_cmd "test -f /home/cjennings/$config"; then + missing="$missing $config" + fi + done + + if [ -z "$missing" ]; then + validation_pass "All Hyprland config files present" + else + validation_fail "Missing Hyprland configs:$missing" + attribute_issue "Missing Hyprland configs:$missing" "archsetup" + fi +} + +validate_hyprland_socket() { + step "Checking Hyprland IPC socket" + # Note: This only works if Hyprland is running. Skip if no display. + if ssh_cmd "test -S /tmp/hypr/*/.socket.sock 2>/dev/null"; then + validation_pass "Hyprland socket exists" + else + validation_warn "Hyprland socket not found (Hyprland may not be running)" + fi +} + +validate_window_manager() { + # Detect which desktop environment is installed and validate accordingly + if ssh_cmd "pacman -Q hyprland &>/dev/null"; then + section "Hyprland Desktop Environment" + validate_hyprland_tools + validate_hyprland_config + validate_hyprland_socket + elif ssh_cmd "test -f /usr/local/bin/dwm"; then + section "DWM Desktop Environment" + validate_suckless_tools + else + validation_warn "No window manager detected (DESKTOP_ENV=none?)" + fi +} + +#----------------------------------------------------------------------------- +# Essential Services Validations +#----------------------------------------------------------------------------- + +validate_firewall() { + step "Checking if firewall (ufw) is enabled" + local status=$(ssh_cmd "systemctl is-enabled ufw.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "UFW firewall is enabled" + else + validation_fail "UFW firewall not enabled" + attribute_issue "UFW not enabled" "archsetup" + fi +} + +validate_dns_config() { + step "Checking DNS-over-TLS configuration" + if ssh_cmd "grep -q 'DNS=.*#' /etc/systemd/resolved.conf 2>/dev/null"; then + validation_pass "DNS-over-TLS configured" + else + validation_warn "DNS-over-TLS may not be configured" + fi +} + +validate_avahi() { + step "Checking avahi-daemon status" + local status=$(ssh_cmd "systemctl is-enabled avahi-daemon.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "avahi-daemon is enabled" + + # Full-stack mDNS test: ping hostname.local + local hostname=$(ssh_cmd "hostname") + if ssh_cmd "ping -c 1 -W 2 ${hostname}.local" &>> "$LOGFILE"; then + validation_pass "mDNS working (${hostname}.local responds to ping)" + else + validation_warn "mDNS ping failed (avahi may need time to propagate)" + fi + else + # This might be OK if avahi was pre-installed + validation_warn "avahi-daemon not enabled (may have been pre-configured)" + fi +} + +validate_fail2ban() { + step "Checking fail2ban status" + local status=$(ssh_cmd "systemctl is-enabled fail2ban.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "fail2ban is enabled" + else + validation_fail "fail2ban not enabled" + attribute_issue "fail2ban not enabled" "archsetup" + fi +} + +validate_networkmanager() { + step "Checking NetworkManager status" + local status=$(ssh_cmd "systemctl is-enabled NetworkManager.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "NetworkManager is enabled" + # Functional test + if ssh_cmd "nmcli general status" &>> "$LOGFILE"; then + validation_pass "NetworkManager is functional" + else + validation_warn "NetworkManager enabled but not responding" + fi + else + validation_fail "NetworkManager not enabled" + attribute_issue "NetworkManager not enabled" "archsetup" + fi +} + +#----------------------------------------------------------------------------- +# Service-Specific Validations +#----------------------------------------------------------------------------- + +validate_all_services() { + section "Service Validations" + + # Core services (always expected) + validate_service "sshd" "enabled" "active" + validate_service "systemd-resolved" "enabled" "active" + validate_service "ufw" "enabled" "" # VM lacks iptables modules, can't be active + validate_service "fail2ban" "enabled" "active" + validate_service "NetworkManager" "enabled" "active" + validate_service "rngd" "enabled" "active" + validate_service "cronie" "enabled" "" + validate_service "atd" "enabled" "" + + # Timer services + validate_service "reflector.timer" "enabled" "" + validate_service "paccache.timer" "enabled" "" + + # Optional services (warn if missing, don't fail) + validate_service_optional "avahi-daemon" "enabled" + validate_service_optional "bluetooth" "enabled" + validate_service_optional "cups" "enabled" + validate_service_optional "docker" "enabled" + validate_service_optional "tailscaled" "enabled" + # Syncthing uses user service (not system), check lingering is enabled + step "Checking user lingering for syncthing" + local linger_enabled=$(ssh_cmd "ls /var/lib/systemd/linger/cjennings 2>/dev/null && echo yes || echo no") + if [ "$linger_enabled" = "yes" ]; then + validation_pass "User lingering enabled for syncthing user service" + else + validation_warn "User lingering not enabled (syncthing may not autostart)" + fi + + # Filesystem-specific + validate_zfs_services + validate_btrfs_services + + # Functional tests + validate_service_functions +} + +validate_service() { + local service="$1" + local expected_enabled="$2" # "enabled" or "" + local expected_active="$3" # "active" or "" + + step "Checking $service" + + if [ -n "$expected_enabled" ]; then + local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled") + if [ "$enabled" = "enabled" ]; then + validation_pass "$service is enabled" + else + validation_fail "$service not enabled (got: $enabled)" + attribute_issue "$service not enabled" "archsetup" + return 1 + fi + fi + + if [ -n "$expected_active" ]; then + local active=$(ssh_cmd "systemctl is-active $service 2>/dev/null || echo inactive") + if [ "$active" = "active" ]; then + validation_pass "$service is active" + else + validation_fail "$service not active (got: $active)" + attribute_issue "$service not active" "archsetup" + return 1 + fi + fi + + return 0 +} + +validate_service_optional() { + local service="$1" + local expected_enabled="$2" + + step "Checking optional service: $service" + + local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled") + if [ "$enabled" = "enabled" ]; then + validation_pass "$service is enabled" + else + validation_warn "$service not enabled (optional)" + fi +} + +validate_zfs_services() { + # Only check if ZFS is installed + if ! ssh_cmd "which zfs" &>> "$LOGFILE"; then + return 0 + fi + + step "Checking ZFS-specific services" + + validate_service_optional "sanoid.timer" "enabled" + + # Check for zfs-scrub timer (pool name varies) + local scrub_enabled + scrub_enabled=$(ssh_cmd "systemctl list-unit-files 'zfs-scrub*' 2>/dev/null | grep -c enabled" | tr -d '[:space:]') + scrub_enabled=${scrub_enabled:-0} + if [ "$scrub_enabled" -gt 0 ]; then + validation_pass "ZFS scrub timer enabled" + else + validation_warn "ZFS scrub timer not found" + fi +} + +validate_btrfs_services() { + # Only check if btrfs root + if ! ssh_cmd "mount | grep 'on / ' | grep -q btrfs"; then + return 0 + fi + + step "Checking btrfs-specific services" + validate_service_optional "grub-btrfsd" "enabled" +} + +validate_service_functions() { + section "Service Functional Tests" + + # UFW functional test + # NOTE: VM environment lacks iptables kernel modules, so UFW cannot activate. + # We only verify it's enabled; active status requires real hardware. + step "Testing UFW functionality" + local ufw_enabled + ufw_enabled=$(ssh_cmd "systemctl is-enabled ufw.service 2>/dev/null || echo disabled") + if [ "$ufw_enabled" = "enabled" ]; then + validation_pass "UFW is enabled (activation requires iptables kernel modules)" + else + validation_fail "UFW not enabled" + attribute_issue "UFW not enabled" "archsetup" + fi + + # fail2ban functional test + step "Testing fail2ban functionality" + if ssh_cmd "fail2ban-client status" &>> "$LOGFILE"; then + validation_pass "fail2ban is responding" + else + validation_fail "fail2ban not responding" + attribute_issue "fail2ban not functioning" "archsetup" + fi + + # DNS resolution test + step "Testing DNS resolution" + if ssh_cmd "resolvectl query archlinux.org" &>> "$LOGFILE"; then + validation_pass "DNS resolution working" + else + validation_warn "DNS resolution test failed (may be network issue)" + fi + + # Docker functional test (if enabled) + if ssh_cmd "systemctl is-enabled docker" &>> "$LOGFILE"; then + step "Testing Docker functionality" + if ssh_cmd "docker info" &>> "$LOGFILE"; then + validation_pass "Docker is responding" + else + validation_warn "Docker enabled but not responding" + fi + fi +} + +#----------------------------------------------------------------------------- +# Developer Tools Validations +#----------------------------------------------------------------------------- + +validate_emacs() { + step "Checking if Emacs is installed" + if ssh_cmd "which emacs" &>> "$LOGFILE"; then + validation_pass "Emacs is installed" + + # Check if config exists + if ssh_cmd "test -d /home/cjennings/.emacs.d"; then + validation_pass "Emacs config directory exists" + + # Check user can access the directory + local result + result=$(ssh_cmd "sudo -u cjennings ls /home/cjennings/.emacs.d > /dev/null 2>&1 && echo OK || echo FAIL") + if [ "$result" = "OK" ]; then + validation_pass "Emacs config readable by user" + else + validation_fail "Emacs config not readable by user (permission issue)" + attribute_issue "Emacs .emacs.d not readable by user" "archsetup" + fi + else + validation_warn "Emacs config directory not found" + fi + else + validation_fail "Emacs not found" + attribute_issue "Emacs not installed" "archsetup" + fi +} + +validate_git_config() { + step "Checking git installation" + if ssh_cmd "which git" &>> "$LOGFILE"; then + validation_pass "git is installed" + else + validation_fail "git not found" + attribute_issue "git not installed" "archsetup" + fi +} + +validate_dev_tools() { + step "Checking developer tools" + local tools="python node npm go rustc" + local missing="" + + for tool in $tools; do + if ! ssh_cmd "which $tool" &>> "$LOGFILE"; then + missing="$missing $tool" + fi + done + + if [ -z "$missing" ]; then + validation_pass "Core dev tools installed" + else + validation_warn "Some dev tools missing:$missing" + fi +} + +#----------------------------------------------------------------------------- +# System Configuration Validations +#----------------------------------------------------------------------------- + +validate_zfs_config() { + step "Checking ZFS configuration (if applicable)" + if ssh_cmd "which zfs" &>> "$LOGFILE"; then + # ZFS is installed, check for sanoid + if ssh_cmd "which sanoid" &>> "$LOGFILE"; then + validation_pass "ZFS with sanoid detected" + else + validation_warn "ZFS detected but sanoid not installed" + fi + else + info "ZFS not installed (non-ZFS system)" + fi +} + +validate_boot_config() { + step "Checking GRUB configuration" + if ssh_cmd "test -f /boot/grub/grub.cfg" &>> "$LOGFILE"; then + validation_pass "GRUB config exists" + else + validation_warn "GRUB config not found (may use different bootloader)" + fi +} + +validate_terminus_font() { + step "Checking terminus-font installation" + if ssh_cmd "pacman -Q terminus-font" &>> "$LOGFILE"; then + validation_pass "terminus-font package installed" + else + validation_fail "terminus-font package not installed" + attribute_issue "terminus-font not installed via pacman" "archsetup" + fi +} + +validate_mkinitcpio_hooks() { + step "Checking mkinitcpio HOOKS configuration" + local hooks=$(ssh_cmd "grep '^HOOKS=' /etc/mkinitcpio.conf") + local is_zfs=$(ssh_cmd "findmnt -n -o FSTYPE / 2>/dev/null") + + if [ "$is_zfs" = "zfs" ]; then + # ZFS system: must use udev, not systemd + if echo "$hooks" | grep -q '\budev\b'; then + validation_pass "ZFS system uses udev hook (correct)" + elif echo "$hooks" | grep -q '\bsystemd\b'; then + validation_fail "ZFS system uses systemd hook (will break boot)" + attribute_issue "mkinitcpio uses systemd hook on ZFS system" "archsetup" + else + validation_warn "Could not determine init hook type" + fi + else + # Non-ZFS: systemd hook is fine + if echo "$hooks" | grep -q '\bsystemd\b'; then + validation_pass "Non-ZFS system uses systemd hook" + elif echo "$hooks" | grep -q '\budev\b'; then + validation_pass "Non-ZFS system uses udev hook" + fi + fi +} + +validate_initramfs_consolefont() { + step "Checking console font in initramfs" + local font_in_initramfs=$(ssh_cmd "lsinitcpio /boot/initramfs-linux*.img 2>/dev/null | grep -c 'consolefont.psf\\|ter-'") + + if [ "${font_in_initramfs:-0}" -gt 0 ]; then + validation_pass "Console font included in initramfs" + else + validation_warn "Console font may not be in initramfs" + fi +} + +validate_nvme_module() { + step "Checking NVMe module configuration" + local has_nvme=$(ssh_cmd "ls /dev/nvme* 2>/dev/null | head -1") + + if [ -n "$has_nvme" ]; then + # System has NVMe drives + local modules=$(ssh_cmd "grep '^MODULES=' /etc/mkinitcpio.conf") + if echo "$modules" | grep -q 'nvme'; then + validation_pass "NVMe module in mkinitcpio MODULES" + else + validation_warn "NVMe system but nvme not in MODULES (may cause slow boot)" + fi + else + info "No NVMe drives detected" + fi +} + +validate_autologin_config() { + step "Checking autologin configuration" + if ssh_cmd "test -f /etc/systemd/system/getty@tty1.service.d/autologin.conf" &>> "$LOGFILE"; then + validation_pass "Autologin configured" + else + info "Autologin not configured (may be intentional)" + fi +} + +validate_gnome_keyring_setup() { + step "Checking gnome-keyring pre-configuration" + local keyring_dir="/home/cjennings/.local/share/keyrings" + + # Check directory exists + if ! ssh_cmd "test -d $keyring_dir"; then + validation_fail "Keyring directory not created" + attribute_issue "gnome-keyring directory not pre-created" "archsetup" + return 1 + fi + + # Check directory permissions (should be 700) + local perms=$(ssh_cmd "stat -c '%a' $keyring_dir") + if [ "$perms" != "700" ]; then + validation_fail "Keyring directory has wrong permissions: $perms (expected 700)" + attribute_issue "gnome-keyring directory wrong permissions" "archsetup" + return 1 + fi + + # Check ownership + local owner=$(ssh_cmd "stat -c '%U' $keyring_dir") + if [ "$owner" != "cjennings" ]; then + validation_fail "Keyring directory owned by $owner (expected cjennings)" + attribute_issue "gnome-keyring directory wrong ownership" "archsetup" + return 1 + fi + + # Check default file exists and contains "login" + local default_keyring=$(ssh_cmd "cat $keyring_dir/default 2>/dev/null") + if [ "$default_keyring" != "login" ]; then + validation_fail "Default keyring not set to 'login' (got: '$default_keyring')" + attribute_issue "gnome-keyring default not set to login" "archsetup" + return 1 + fi + + validation_pass "gnome-keyring pre-configured (default=login, dir=700)" +} + +#----------------------------------------------------------------------------- +# Archsetup-Specific Validations +#----------------------------------------------------------------------------- + +validate_archsetup_log() { + step "Checking archsetup log for errors" + local error_count + # Use grep -h to suppress filenames, then wc -l to count total matches + error_count=$(ssh_cmd "grep -h '^Error:' /var/log/archsetup-*.log 2>/dev/null | wc -l" | tr -d '[:space:]') + error_count=${error_count:-0} + + if [ "$error_count" = "0" ]; then + validation_pass "No errors in archsetup log" + else + validation_fail "Found $error_count errors in archsetup log" + attribute_issue "Errors in archsetup log: $error_count" "archsetup" + fi +} + +validate_state_markers() { + step "Checking archsetup state markers" + local state_count=$(ssh_cmd "ls /var/lib/archsetup/state/ 2>/dev/null | wc -l") + + if [ "$state_count" -ge 12 ]; then + validation_pass "All 12 installation steps completed" + else + validation_warn "Only $state_count/12 steps completed" + fi +} + +#============================================================================= +# ISSUE REPORTING +#============================================================================= + +generate_issue_report() { + local output_dir="$1" + local archzfs_inbox="$2" + + section "Issue Attribution Report" + + local report_file="$output_dir/issue-report.txt" + + cat > "$report_file" << EOF +======================================== +Issue Attribution Report +Generated: $(date +'%Y-%m-%d %H:%M:%S') +======================================== + +VALIDATION SUMMARY +------------------ +Passed: $VALIDATION_PASSED +Failed: $VALIDATION_FAILED +Warnings: $VALIDATION_WARNINGS + +EOF + + if [ ${#ARCHSETUP_ISSUES[@]} -gt 0 ]; then + echo "ARCHSETUP ISSUES (${#ARCHSETUP_ISSUES[@]})" >> "$report_file" + echo "-------------------------------------------" >> "$report_file" + for issue in "${ARCHSETUP_ISSUES[@]}"; do + echo " - $issue" >> "$report_file" + done + echo "" >> "$report_file" + + error "Found ${#ARCHSETUP_ISSUES[@]} archsetup issues" + fi + + if [ ${#BASE_INSTALL_ISSUES[@]} -gt 0 ]; then + echo "BASE INSTALL ISSUES (${#BASE_INSTALL_ISSUES[@]})" >> "$report_file" + echo "-------------------------------------------" >> "$report_file" + for issue in "${BASE_INSTALL_ISSUES[@]}"; do + echo " - $issue" >> "$report_file" + done + echo "" >> "$report_file" + + warn "Found ${#BASE_INSTALL_ISSUES[@]} base install issues" + + # If archzfs inbox provided, create issue files + if [ -n "$archzfs_inbox" ] && [ -d "$archzfs_inbox" ]; then + local issue_file="$archzfs_inbox/$(date +'%Y-%m-%d')-test-issues.txt" + echo "Base install issues from archsetup test run:" > "$issue_file" + echo "Date: $(date +'%Y-%m-%d %H:%M:%S')" >> "$issue_file" + echo "" >> "$issue_file" + for issue in "${BASE_INSTALL_ISSUES[@]}"; do + echo "- $issue" >> "$issue_file" + done + info "Created archzfs issue file: $issue_file" + fi + fi + + if [ ${#UNKNOWN_ISSUES[@]} -gt 0 ]; then + echo "UNKNOWN/UNATTRIBUTED ISSUES (${#UNKNOWN_ISSUES[@]})" >> "$report_file" + echo "-------------------------------------------" >> "$report_file" + for issue in "${UNKNOWN_ISSUES[@]}"; do + echo " - $issue" >> "$report_file" + done + echo "" >> "$report_file" + + warn "Found ${#UNKNOWN_ISSUES[@]} unattributed issues" + fi + + if [ ${#ARCHSETUP_ISSUES[@]} -eq 0 ] && [ ${#BASE_INSTALL_ISSUES[@]} -eq 0 ] && [ ${#UNKNOWN_ISSUES[@]} -eq 0 ]; then + echo "No issues found!" >> "$report_file" + success "No issues found!" + fi + + info "Issue report saved: $report_file" +} + +#============================================================================= +# MAIN VALIDATION ENTRY POINT +#============================================================================= + +run_full_validation() { + local output_dir="$1" + local archzfs_inbox="${2:-}" + + run_all_validations + analyze_log_diff "$output_dir" + generate_issue_report "$output_dir" "$archzfs_inbox" + + # Return success if no failures + [ $VALIDATION_FAILED -eq 0 ] +} diff --git a/scripts/testing/lib/vm-utils.sh b/scripts/testing/lib/vm-utils.sh new file mode 100755 index 0000000..81aec33 --- /dev/null +++ b/scripts/testing/lib/vm-utils.sh @@ -0,0 +1,321 @@ +#!/bin/bash +# VM management utilities for archsetup testing +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 + +# Note: logging.sh should already be sourced by the calling script + +# VM configuration defaults +VM_CPUS="${VM_CPUS:-4}" +VM_RAM="${VM_RAM:-8192}" # MB +VM_DISK="${VM_DISK:-50}" # GB +VM_NETWORK="${VM_NETWORK:-default}" +LIBVIRT_URI="qemu:///system" # Use system session, not user session + +# Check if libvirt is running +check_libvirt() { + if ! systemctl is-active --quiet libvirtd; then + error "libvirtd service is not running" + info "Start it with: sudo systemctl start libvirtd" + return 1 + fi + return 0 +} + +# Check if user is in libvirt group +check_libvirt_group() { + if ! groups | grep -q libvirt; then + warn "Current user is not in libvirt group" + info "Add yourself with: sudo usermod -a -G libvirt $USER" + info "Then log out and back in for changes to take effect" + return 1 + fi + return 0 +} + +# Check if KVM is available +check_kvm() { + if [ ! -e /dev/kvm ]; then + error "KVM is not available" + info "Check if virtualization is enabled in BIOS" + info "Load kvm module: sudo modprobe kvm-intel (or kvm-amd)" + return 1 + fi + return 0 +} + +# Wait for VM to boot (check for SSH or serial console) +wait_for_vm() { + local vm_name="$1" + local timeout="${2:-300}" # 5 minutes default + local elapsed=0 + + progress "Waiting for VM $vm_name to boot..." + + while [ $elapsed -lt $timeout ]; do + if virsh --connect "$LIBVIRT_URI" domstate "$vm_name" 2>/dev/null | grep -q "running"; then + sleep 5 + complete "VM $vm_name is running" + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + + error "Timeout waiting for VM $vm_name to boot" + return 1 +} + +# Check if VM exists +vm_exists() { + local vm_name="$1" + virsh --connect "$LIBVIRT_URI" dominfo "$vm_name" &>/dev/null + return $? +} + +# Check if VM is running +vm_is_running() { + local vm_name="$1" + [ "$(virsh --connect "$LIBVIRT_URI" domstate "$vm_name" 2>/dev/null)" = "running" ] + return $? +} + +# Start VM +start_vm() { + local vm_name="$1" + + if vm_is_running "$vm_name"; then + warn "VM $vm_name is already running" + return 0 + fi + + step "Starting VM: $vm_name" + if virsh --connect "$LIBVIRT_URI" start "$vm_name" >> "$LOGFILE" 2>&1; then + success "VM $vm_name started" + return 0 + else + error "Failed to start VM $vm_name" + return 1 + fi +} + +# Stop VM gracefully +stop_vm() { + local vm_name="$1" + local timeout="${2:-60}" + + if ! vm_is_running "$vm_name"; then + info "VM $vm_name is not running" + return 0 + fi + + step "Shutting down VM: $vm_name" + if virsh --connect "$LIBVIRT_URI" shutdown "$vm_name" >> "$LOGFILE" 2>&1; then + # Wait for graceful shutdown + local elapsed=0 + while [ $elapsed -lt $timeout ]; do + if ! vm_is_running "$vm_name"; then + success "VM $vm_name stopped gracefully" + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + + warn "VM $vm_name did not stop gracefully, forcing..." + virsh --connect "$LIBVIRT_URI" destroy "$vm_name" >> "$LOGFILE" 2>&1 + fi + + success "VM $vm_name stopped" + return 0 +} + +# Destroy VM (force stop) +destroy_vm() { + local vm_name="$1" + + if ! vm_exists "$vm_name"; then + info "VM $vm_name does not exist" + return 0 + fi + + step "Destroying VM: $vm_name" + if vm_is_running "$vm_name"; then + virsh --connect "$LIBVIRT_URI" destroy "$vm_name" >> "$LOGFILE" 2>&1 + fi + + virsh --connect "$LIBVIRT_URI" undefine "$vm_name" --nvram >> "$LOGFILE" 2>&1 + success "VM $vm_name destroyed" + return 0 +} + +# Create snapshot +create_snapshot() { + local vm_name="$1" + local snapshot_name="$2" + + step "Creating snapshot: $snapshot_name" + if virsh --connect "$LIBVIRT_URI" snapshot-create-as "$vm_name" "$snapshot_name" >> "$LOGFILE" 2>&1; then + success "Snapshot $snapshot_name created" + return 0 + else + error "Failed to create snapshot $snapshot_name" + return 1 + fi +} + +# Restore snapshot +restore_snapshot() { + local vm_name="$1" + local snapshot_name="$2" + + step "Restoring snapshot: $snapshot_name" + if virsh --connect "$LIBVIRT_URI" snapshot-revert "$vm_name" "$snapshot_name" >> "$LOGFILE" 2>&1; then + success "Snapshot $snapshot_name restored" + return 0 + else + error "Failed to restore snapshot $snapshot_name" + return 1 + fi +} + +# Delete snapshot +delete_snapshot() { + local vm_name="$1" + local snapshot_name="$2" + + step "Deleting snapshot: $snapshot_name" + if virsh --connect "$LIBVIRT_URI" snapshot-delete "$vm_name" "$snapshot_name" >> "$LOGFILE" 2>&1; then + success "Snapshot $snapshot_name deleted" + return 0 + else + error "Failed to delete snapshot $snapshot_name" + return 1 + fi +} + +# Clone disk image (copy-on-write) +clone_disk() { + local base_image="$1" + local new_image="$2" + + if [ ! -f "$base_image" ]; then + error "Base image not found: $base_image" + return 1 + fi + + step "Cloning disk image (full copy)" + if qemu-img convert -f qcow2 -O qcow2 "$base_image" "$new_image" >> "$LOGFILE" 2>&1; then + success "Disk cloned: $new_image" + else + error "Failed to clone disk" + return 1 + fi + + # Truncate machine-id so systemd generates a new one on boot (avoids DHCP conflicts) + step "Clearing machine-id for unique network identity" + if guestfish -a "$new_image" -i truncate /etc/machine-id >> "$LOGFILE" 2>&1; then + success "Machine-ID cleared (will regenerate on boot)" + return 0 + else + warn "Failed to clear machine-ID (guestfish failed)" + info "Network may conflict with base VM if both run simultaneously" + return 0 # Don't fail the whole operation + fi +} + +# Get VM IP address (requires guest agent or DHCP lease) +get_vm_ip() { + local vm_name="$1" + + # Try guest agent first + local ip + ip=$(virsh --connect "$LIBVIRT_URI" domifaddr "$vm_name" 2>/dev/null | grep -oP '(\d+\.){3}\d+' | head -1) + + if [ -n "$ip" ]; then + echo "$ip" + return 0 + fi + + # Fall back to DHCP leases + local mac + mac=$(virsh --connect "$LIBVIRT_URI" domiflist "$vm_name" | grep -oP '([0-9a-f]{2}:){5}[0-9a-f]{2}' | head -1) + + if [ -n "$mac" ]; then + ip=$(grep "$mac" /var/lib/libvirt/dnsmasq/default.leases 2>/dev/null | awk '{print $3}') + if [ -n "$ip" ]; then + echo "$ip" + return 0 + fi + fi + + return 1 +} + +# Execute command in VM via SSH +vm_exec() { + local vm_name="$1" + shift + local cmd="$*" + + local ip + ip=$(get_vm_ip "$vm_name") + + if [ -z "$ip" ]; then + error "Could not get IP address for VM $vm_name" + return 1 + fi + + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$ip" "$cmd" 2>> "$LOGFILE" +} + +# Copy file to VM +copy_to_vm() { + local vm_name="$1" + local local_file="$2" + local remote_path="$3" + + local ip + ip=$(get_vm_ip "$vm_name") + + if [ -z "$ip" ]; then + error "Could not get IP address for VM $vm_name" + return 1 + fi + + step "Copying $local_file to VM" + if scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "$local_file" "root@$ip:$remote_path" >> "$LOGFILE" 2>&1; then + success "File copied to VM" + return 0 + else + error "Failed to copy file to VM" + return 1 + fi +} + +# Copy file from VM +copy_from_vm() { + local vm_name="$1" + local remote_file="$2" + local local_path="$3" + + local ip + ip=$(get_vm_ip "$vm_name") + + if [ -z "$ip" ]; then + error "Could not get IP address for VM $vm_name" + return 1 + fi + + step "Copying $remote_file from VM" + if scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$ip:$remote_file" "$local_path" >> "$LOGFILE" 2>&1; then + success "File copied from VM" + return 0 + else + error "Failed to copy file from VM" + return 1 + fi +} diff --git a/scripts/testing/run-test-baremetal.sh b/scripts/testing/run-test-baremetal.sh new file mode 100755 index 0000000..c108e6f --- /dev/null +++ b/scripts/testing/run-test-baremetal.sh @@ -0,0 +1,310 @@ +#!/bin/bash +# Run archsetup test on bare metal ZFS system +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# This script: +# 1. Connects to bare metal ZFS system via SSH +# 2. Optionally rolls back to genesis snapshots first +# 3. Transfers archsetup +# 4. Executes archsetup +# 5. Captures logs and validates results +# 6. Can rollback to genesis if test fails + +set -e + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source utilities +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/validation.sh" + +# Parse arguments +ROLLBACK_FIRST=false +ROLLBACK_AFTER=false +TARGET_HOST="" +ROOT_PASSWORD="" + +usage() { + echo "Usage: $0 --host <hostname> --password <root_password> [options]" + echo "" + echo "Required:" + echo " --host <hostname> Target bare metal host (e.g., ratio.local)" + echo " --password <password> Root password for SSH" + echo "" + echo "Options:" + echo " --rollback-first Roll back to genesis snapshots before running" + echo " --rollback-after Roll back to genesis snapshots after test (cleanup)" + echo " --validate-only Skip archsetup, only run validation checks" + echo " --help Show this help" + exit 1 +} + +VALIDATE_ONLY=false + +while [[ $# -gt 0 ]]; do + case $1 in + --host) + TARGET_HOST="$2" + shift 2 + ;; + --password) + ROOT_PASSWORD="$2" + shift 2 + ;; + --rollback-first) + ROLLBACK_FIRST=true + shift + ;; + --rollback-after) + ROLLBACK_AFTER=true + shift + ;; + --validate-only) + VALIDATE_ONLY=true + shift + ;; + --help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +# Validate required args +if [ -z "$TARGET_HOST" ] || [ -z "$ROOT_PASSWORD" ]; then + echo "Error: --host and --password are required" + usage +fi + +# Configuration +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +TEST_RESULTS_DIR="$PROJECT_ROOT/test-results/baremetal-$TIMESTAMP" +ARCHZFS_INBOX="$HOME/code/archzfs/inbox" + +# Override VM_IP for validation.sh ssh_cmd function +VM_IP="$TARGET_HOST" + +# Initialize logging +mkdir -p "$TEST_RESULTS_DIR" +LOGFILE="$TEST_RESULTS_DIR/test.log" +init_logging "$LOGFILE" + +section "Bare Metal Test Run: $TIMESTAMP" +info "Target: $TARGET_HOST" + +# Test SSH connectivity +step "Testing SSH connectivity to $TARGET_HOST" +if ! sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 "root@$TARGET_HOST" "echo connected" &>/dev/null; then + fatal "Cannot connect to $TARGET_HOST via SSH" +fi +success "SSH connection OK" + +# Check it's a ZFS system +step "Verifying ZFS root" +if ! ssh_cmd "zfs list zroot" &>> "$LOGFILE"; then + fatal "Target is not a ZFS system (no zroot pool)" +fi +success "ZFS root confirmed" + +# Rollback to genesis if requested +if $ROLLBACK_FIRST; then + section "Rolling Back to Genesis Snapshots" + + step "Getting list of datasets with genesis snapshots" + DATASETS=$(ssh_cmd "zfs list -H -o name -t snapshot | grep '@genesis$' | sed 's/@genesis$//'") + + step "Rolling back all datasets to genesis" + for ds in $DATASETS; do + info "Rolling back $ds@genesis" + if ! ssh_cmd "zfs rollback -r $ds@genesis" &>> "$LOGFILE"; then + warn "Failed to rollback $ds@genesis" + fi + done + success "Rollback complete" + + # Need to reconnect after rollback + sleep 5 + step "Reconnecting after rollback" + if ! ssh_cmd "echo reconnected" &>/dev/null; then + fatal "Lost connection after rollback" + fi + success "Reconnected" +fi + +if ! $VALIDATE_ONLY; then + # Capture pre-install state + capture_pre_install_state "$TEST_RESULTS_DIR" + + # Transfer archsetup + section "Transferring ArchSetup" + + step "Creating git bundle" + BUNDLE_FILE=$(mktemp) + git -C "$PROJECT_ROOT" bundle create "$BUNDLE_FILE" HEAD >> "$LOGFILE" 2>&1 + + step "Transferring to $TARGET_HOST" + ssh_cmd "rm -rf /tmp/archsetup-test && mkdir -p /tmp/archsetup-test" + sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "$BUNDLE_FILE" "root@$TARGET_HOST:/tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 + + step "Extracting on target" + ssh_cmd "cd /tmp && git clone /tmp/archsetup.bundle archsetup-test && rm /tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 + rm -f "$BUNDLE_FILE" + success "ArchSetup transferred" + + # Execute archsetup + section "Executing ArchSetup" + + start_timer "archsetup" + step "Starting archsetup on $TARGET_HOST" + info "This will take 30-60 minutes" + + REMOTE_LOG="/tmp/archsetup-test/archsetup-output.log" + + # Start archsetup in background + ssh_cmd "cd /tmp/archsetup-test && nohup bash archsetup > $REMOTE_LOG 2>&1 &" + success "ArchSetup started in background" + + # Poll for completion + step "Monitoring archsetup progress" + POLL_COUNT=0 + MAX_POLLS=180 # 90 minutes max + + while [ $POLL_COUNT -lt $MAX_POLLS ]; do + if ssh_cmd "ps aux | grep '[b]ash archsetup' > /dev/null" 2>/dev/null; then + sleep 30 + POLL_COUNT=$((POLL_COUNT + 1)) + if [ $((POLL_COUNT % 10)) -eq 0 ]; then + ELAPSED_MINS=$((POLL_COUNT / 2)) + info "Still running... ($ELAPSED_MINS minutes elapsed)" + # Show last line of progress + LAST_LINE=$(ssh_cmd "tail -1 $REMOTE_LOG 2>/dev/null" || echo "") + [ -n "$LAST_LINE" ] && info " $LAST_LINE" + fi + else + break + fi + done + + if [ $POLL_COUNT -ge $MAX_POLLS ]; then + error "ArchSetup timed out after 90 minutes" + ARCHSETUP_EXIT_CODE=124 + else + step "Retrieving archsetup exit status" + if ssh_cmd "grep -q 'ARCHSETUP_EXECUTION_COMPLETE' /var/log/archsetup-*.log 2>/dev/null"; then + ARCHSETUP_EXIT_CODE=0 + success "ArchSetup completed successfully" + else + ARCHSETUP_EXIT_CODE=1 + error "ArchSetup may have encountered errors" + fi + fi + + stop_timer "archsetup" + + # Copy logs + section "Capturing Test Artifacts" + + step "Copying archsetup log" + sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$TARGET_HOST:/var/log/archsetup-*.log" "$TEST_RESULTS_DIR/" 2>> "$LOGFILE" || \ + warn "Could not copy archsetup log" + + step "Copying archsetup output" + sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$TARGET_HOST:$REMOTE_LOG" "$TEST_RESULTS_DIR/archsetup-output.log" 2>> "$LOGFILE" || \ + warn "Could not copy output log" + + # Capture post-install state + capture_post_install_state "$TEST_RESULTS_DIR" +else + info "Skipping archsetup (--validate-only)" + ARCHSETUP_EXIT_CODE=0 + mkdir -p "$TEST_RESULTS_DIR/pre-install" "$TEST_RESULTS_DIR/post-install" +fi + +# Run validations +run_all_validations +validate_all_services + +# Additional ZFS-specific validations +section "ZFS-Specific Validations" +validate_zfs_services + +# Analyze logs if we ran archsetup +if ! $VALIDATE_ONLY; then + analyze_log_diff "$TEST_RESULTS_DIR" +fi + +# Generate reports +generate_issue_report "$TEST_RESULTS_DIR" "$ARCHZFS_INBOX" + +# Set validation result +if [ $VALIDATION_FAILED -eq 0 ]; then + VALIDATION_PASSED=true +else + VALIDATION_PASSED=false +fi + +# Generate test report +section "Generating Test Report" + +REPORT_FILE="$TEST_RESULTS_DIR/test-report.txt" +cat > "$REPORT_FILE" << EOFREPORT +======================================== +Bare Metal ArchSetup Test Report +======================================== + +Test ID: $TIMESTAMP +Date: $(date +'%Y-%m-%d %H:%M:%S') +Target: $TARGET_HOST +Test Method: Bare Metal ZFS + +Results: + ArchSetup Exit Code: $ARCHSETUP_EXIT_CODE + Validation: $(if $VALIDATION_PASSED; then echo "PASSED"; else echo "FAILED"; fi) + +Validation Summary: + Passed: $VALIDATION_PASSED_COUNT + Failed: $VALIDATION_FAILED + Warnings: $VALIDATION_WARNINGS + +Artifacts: + Log file: $LOGFILE + Report: $REPORT_FILE + Results: $TEST_RESULTS_DIR/ + +EOFREPORT + +info "Test report saved: $REPORT_FILE" + +# Rollback after if requested +if $ROLLBACK_AFTER; then + section "Rolling Back to Genesis (cleanup)" + + DATASETS=$(ssh_cmd "zfs list -H -o name -t snapshot | grep '@genesis$' | sed 's/@genesis$//'") + for ds in $DATASETS; do + info "Rolling back $ds@genesis" + ssh_cmd "zfs rollback -r $ds@genesis" &>> "$LOGFILE" || true + done + success "Rollback complete" +fi + +# Final summary +section "Test Complete" + +if [ $ARCHSETUP_EXIT_CODE -eq 0 ] && $VALIDATION_PASSED; then + success "TEST PASSED" + exit 0 +else + error "TEST FAILED" + info "Check logs in: $TEST_RESULTS_DIR" + exit 1 +fi diff --git a/scripts/testing/run-test.sh b/scripts/testing/run-test.sh new file mode 100755 index 0000000..4c41cc3 --- /dev/null +++ b/scripts/testing/run-test.sh @@ -0,0 +1,367 @@ +#!/bin/bash +# Run archsetup test in a VM using snapshots +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# This script: +# 1. Reverts base VM to clean snapshot +# 2. Starts the base VM +# 3. Transfers archsetup and dotfiles +# 4. Executes archsetup in the VM +# 5. Captures logs and validates results +# 6. Generates test report +# 7. Reverts to clean snapshot for next run + +set -e + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source utilities +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" +source "$SCRIPT_DIR/lib/network-diagnostics.sh" +source "$SCRIPT_DIR/lib/validation.sh" + +# Parse arguments +KEEP_VM=false +ARCHSETUP_SCRIPT="$PROJECT_ROOT/archsetup" +SNAPSHOT_NAME="clean-install" + +while [[ $# -gt 0 ]]; do + case $1 in + --keep) + KEEP_VM=true + shift + ;; + --script) + ARCHSETUP_SCRIPT="$2" + shift 2 + ;; + --snapshot) + SNAPSHOT_NAME="$2" + shift 2 + ;; + *) + echo "Usage: $0 [--keep] [--script /path/to/archsetup] [--snapshot name]" + echo " --keep Keep VM in post-test state (for debugging)" + echo " --script Specify custom archsetup script to test" + echo " --snapshot Snapshot name to revert to (default: clean-install)" + exit 1 + ;; + esac +done + +# Configuration +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +VM_NAME="archsetup-base" +TEST_RESULTS_DIR="$PROJECT_ROOT/test-results/$TIMESTAMP" +ROOT_PASSWORD="archsetup" +ARCHZFS_INBOX="$HOME/code/archzfs/inbox" + +# Initialize logging +mkdir -p "$TEST_RESULTS_DIR" +LOGFILE="$TEST_RESULTS_DIR/test.log" +init_logging "$LOGFILE" + +section "ArchSetup Test Run: $TIMESTAMP" + +# Verify archsetup script exists +if [ ! -f "$ARCHSETUP_SCRIPT" ]; then + fatal "ArchSetup script not found: $ARCHSETUP_SCRIPT" +fi + +# Check if VM exists +if ! vm_exists "$VM_NAME"; then + fatal "Base VM not found: $VM_NAME" + info "Create it first: ./scripts/testing/create-base-vm.sh" +fi + +# Check if snapshot exists +section "Preparing Test Environment" + +step "Checking for snapshot: $SNAPSHOT_NAME" +if ! virsh --connect "$LIBVIRT_URI" snapshot-list "$VM_NAME" --name 2>/dev/null | grep -q "^$SNAPSHOT_NAME$"; then + fatal "Snapshot '$SNAPSHOT_NAME' not found on VM $VM_NAME" + info "Available snapshots:" + virsh --connect "$LIBVIRT_URI" snapshot-list "$VM_NAME" 2>/dev/null || info " (none)" + info "" + info "Create snapshot with:" + info " virsh snapshot-create-as $VM_NAME $SNAPSHOT_NAME --description 'Clean Arch install'" +fi +success "Snapshot $SNAPSHOT_NAME exists" + +# Shut down VM if running +if vm_is_running "$VM_NAME"; then + warn "VM $VM_NAME is currently running - shutting down for snapshot revert" + stop_vm "$VM_NAME" +fi + +# Revert to clean snapshot +step "Reverting to snapshot: $SNAPSHOT_NAME" +if restore_snapshot "$VM_NAME" "$SNAPSHOT_NAME"; then + success "Reverted to clean state" +else + fatal "Failed to revert snapshot" +fi + +# Start VM +start_timer "boot" +step "Starting VM and waiting for SSH..." +if ! start_vm "$VM_NAME"; then + fatal "Failed to start VM" +fi + +sleep 10 # Give VM time to boot + +# Get VM IP address +VM_IP="" +for i in {1..30}; do + VM_IP=$(get_vm_ip "$VM_NAME" 2>/dev/null || true) + if [ -n "$VM_IP" ]; then + break + fi + sleep 2 +done + +if [ -z "$VM_IP" ]; then + error "Could not get VM IP address" + info "VM may not have booted correctly" + fatal "VM boot failed" +fi + +success "VM is running at $VM_IP" +stop_timer "boot" + +# Wait for SSH +step "Waiting for SSH to become available..." +for i in {1..60}; do + if sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=2 "root@$VM_IP" "echo connected" &>/dev/null; then + break + fi + sleep 2 +done + +success "SSH is available" + +# Run network diagnostics +if ! run_network_diagnostics "$VM_IP"; then + fatal "Network diagnostics failed - aborting test" +fi + +# Capture pre-install state for comparison +capture_pre_install_state "$TEST_RESULTS_DIR" + +# Transfer files to VM (simulating git clone) +section "Simulating Git Clone" + +step "Creating shallow git clone on VM" +info "This simulates: git clone --depth 1 <repo> /home/cjennings/code/archsetup" + +# Create a temporary git bundle from current repo +BUNDLE_FILE=$(mktemp) +git bundle create "$BUNDLE_FILE" HEAD >> "$LOGFILE" 2>&1 + +# Transfer bundle and extract on VM +sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP" "rm -rf /tmp/archsetup-test && mkdir -p /tmp/archsetup-test" >> "$LOGFILE" 2>&1 + +sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "$BUNDLE_FILE" "root@$VM_IP:/tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 + +# Clone from bundle on VM (simulates git clone) +sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP" "cd /tmp && git clone --depth 1 /tmp/archsetup.bundle archsetup-test && rm /tmp/archsetup.bundle" >> "$LOGFILE" 2>&1 + +rm -f "$BUNDLE_FILE" +success "Repository cloned to VM (simulating git clone --depth 1)" + +# Execute archsetup +section "Executing ArchSetup" + +start_timer "archsetup" +step "Starting archsetup script in detached session on VM..." +info "This will take 30-60 minutes depending on network speed" +info "Log file: $LOGFILE" + +# Start archsetup in a detached session on the VM (resilient to SSH disconnections) +REMOTE_LOG="/tmp/archsetup-test/archsetup-output.log" +sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP" "cd /tmp/archsetup-test && nohup bash archsetup > $REMOTE_LOG 2>&1 & echo \$!" \ + >> "$LOGFILE" 2>&1 + +if [ $? -ne 0 ]; then + fatal "Failed to start archsetup on VM" +fi + +success "ArchSetup started in background on VM" + +# Poll for completion +step "Monitoring archsetup progress (polling every 30 seconds)..." +POLL_COUNT=0 +MAX_POLLS=180 # 90 minutes max (180 * 30 seconds) + +while [ $POLL_COUNT -lt $MAX_POLLS ]; do + # Check if archsetup process is still running + # Use ps to avoid pgrep matching its own SSH command + if sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP" "ps aux | grep '[b]ash archsetup' > /dev/null" 2>/dev/null; then + # Still running, wait and continue + sleep 30 + POLL_COUNT=$((POLL_COUNT + 1)) + + # Show progress every 5 minutes + if [ $((POLL_COUNT % 10)) -eq 0 ]; then + ELAPSED_MINS=$((POLL_COUNT / 2)) + info "Still running... ($ELAPSED_MINS minutes elapsed)" + fi + else + # Process finished + break + fi +done + +if [ $POLL_COUNT -ge $MAX_POLLS ]; then + error "ArchSetup timed out after 90 minutes" + ARCHSETUP_EXIT_CODE=124 +else + # Get exit code from the remote log + step "Retrieving archsetup exit status..." + ARCHSETUP_EXIT_CODE=$(sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP" "grep -q 'ARCHSETUP_EXECUTION_COMPLETE' /var/log/archsetup-*.log 2>/dev/null && echo 0 || echo 1" 2>/dev/null) + + if [ "$ARCHSETUP_EXIT_CODE" = "0" ]; then + success "ArchSetup completed successfully" + else + error "ArchSetup may have encountered errors (check logs)" + fi +fi + +# Copy the remote output log +step "Retrieving archsetup output from VM..." +sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP:$REMOTE_LOG" "$TEST_RESULTS_DIR/archsetup-output.log" 2>> "$LOGFILE" || \ + warn "Could not copy remote output log" + +# Append remote output to main test log +if [ -f "$TEST_RESULTS_DIR/archsetup-output.log" ]; then + cat "$TEST_RESULTS_DIR/archsetup-output.log" >> "$LOGFILE" +fi + +stop_timer "archsetup" + +# Capture logs and artifacts from VM +section "Capturing Test Artifacts" + +step "Copying archsetup log from VM" +sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP:/var/log/archsetup-*.log" "$TEST_RESULTS_DIR/" 2>> "$LOGFILE" || \ + warn "Could not copy archsetup log" + +step "Copying package lists from VM" +sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + "root@$VM_IP:/root/.local/src/archsetup-*.txt" "$TEST_RESULTS_DIR/" 2>> "$LOGFILE" || \ + warn "Could not copy package lists" + +# Capture post-install state +capture_post_install_state "$TEST_RESULTS_DIR" + +# Run comprehensive validation +# This uses the validation.sh library for all checks +run_all_validations +validate_all_services + +# Analyze log differences (pre vs post install) +analyze_log_diff "$TEST_RESULTS_DIR" + +# Generate issue attribution report +# If base install issues found and archzfs inbox exists, create issue file +generate_issue_report "$TEST_RESULTS_DIR" "$ARCHZFS_INBOX" + +# Set validation result based on failure count +if [ $VALIDATION_FAILED -eq 0 ]; then + VALIDATION_PASSED=true +else + VALIDATION_PASSED=false +fi + +# Generate test report +section "Generating Test Report" + +REPORT_FILE="$TEST_RESULTS_DIR/test-report.txt" +cat > "$REPORT_FILE" << EOFREPORT +======================================== +ArchSetup Test Report +======================================== + +Test ID: $TIMESTAMP +Date: $(date +'%Y-%m-%d %H:%M:%S') +Test Method: Snapshot-based + +VM Configuration: + Name: $VM_NAME + IP: $VM_IP + Snapshot: $SNAPSHOT_NAME + +Results: + ArchSetup Exit Code: $ARCHSETUP_EXIT_CODE + Validation: $(if $VALIDATION_PASSED; then echo "PASSED"; else echo "FAILED"; fi) + +Validation Summary: + Passed: $VALIDATION_PASSED + Failed: $VALIDATION_FAILED + Warnings: $VALIDATION_WARNINGS + +Issue Attribution: + ArchSetup Issues: ${#ARCHSETUP_ISSUES[@]} + Base Install Issues: ${#BASE_INSTALL_ISSUES[@]} + Unknown Issues: ${#UNKNOWN_ISSUES[@]} + +Artifacts: + Log file: $LOGFILE + Report: $REPORT_FILE + Results: $TEST_RESULTS_DIR/ + Issue Report: $TEST_RESULTS_DIR/issue-report.txt + Pre-install logs: $TEST_RESULTS_DIR/pre-install/ + Post-install logs: $TEST_RESULTS_DIR/post-install/ + Analysis: $TEST_RESULTS_DIR/analysis/ + +EOFREPORT + +info "Test report saved: $REPORT_FILE" + +# Cleanup or keep VM +section "Cleanup" + +if $KEEP_VM; then + info "VM is still running in post-test state (--keep flag was used)" + info "Connect with:" + info " Console: virsh console $VM_NAME" + info " SSH: ssh root@$VM_IP" + info "" + info "To revert to clean state when done:" + info " virsh shutdown $VM_NAME" + info " virsh snapshot-revert $VM_NAME $SNAPSHOT_NAME" +else + step "Shutting down VM and reverting to clean snapshot" + stop_vm "$VM_NAME" + if restore_snapshot "$VM_NAME" "$SNAPSHOT_NAME"; then + success "VM reverted to clean state" + else + warn "Failed to revert snapshot - VM may be in modified state" + fi +fi + +# Final summary +section "Test Complete" + +if [ $ARCHSETUP_EXIT_CODE -eq 0 ] && $VALIDATION_PASSED; then + success "TEST PASSED" + exit 0 +else + error "TEST FAILED" + info "Check logs in: $TEST_RESULTS_DIR" + exit 1 +fi diff --git a/scripts/testing/setup-testing-env.sh b/scripts/testing/setup-testing-env.sh new file mode 100755 index 0000000..e682553 --- /dev/null +++ b/scripts/testing/setup-testing-env.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# Setup testing environment for archsetup +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# This script performs one-time setup of the testing infrastructure: +# - Installs QEMU/KVM and libvirt +# - Configures libvirt networking +# - Adds user to libvirt group +# - Verifies KVM support +# - Creates directories for test artifacts + +set -e + +# Get script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source utilities +source "$SCRIPT_DIR/lib/logging.sh" + +# Initialize logging +LOGFILE="$PROJECT_ROOT/test-results/setup-$(date +'%Y%m%d-%H%M%S').log" +init_logging "$LOGFILE" + +section "ArchSetup Testing Environment Setup" + +# Check if running on Arch Linux +if [ ! -f /etc/arch-release ]; then + fatal "This script is designed for Arch Linux" +fi + +# Check if user has sudo +if ! sudo -n true 2>/dev/null; then + warn "This script requires sudo access" + info "You may be prompted for your password" +fi + +# Install required packages +section "Installing Required Packages" + +PACKAGES=( + qemu-full + libvirt + virt-manager + dnsmasq + bridge-utils + iptables + virt-install + libguestfs +) + +for pkg in "${PACKAGES[@]}"; do + if pacman -Qi "$pkg" &>/dev/null; then + info "$pkg is already installed" + else + step "Installing $pkg" + if sudo pacman -S --noconfirm "$pkg" >> "$LOGFILE" 2>&1; then + success "$pkg installed" + else + error "Failed to install $pkg" + fatal "Package installation failed" + fi + fi +done + +# Enable and start libvirt service +section "Configuring libvirt Service" + +step "Enabling libvirtd service" +if sudo systemctl enable libvirtd.service >> "$LOGFILE" 2>&1; then + success "libvirtd service enabled" +else + warn "Failed to enable libvirtd service (may already be enabled)" +fi + +step "Starting libvirtd service" +if sudo systemctl start libvirtd.service >> "$LOGFILE" 2>&1; then + success "libvirtd service started" +else + if sudo systemctl is-active --quiet libvirtd.service; then + info "libvirtd service is already running" + else + error "Failed to start libvirtd service" + fatal "Service startup failed" + fi +fi + +# Add user to libvirt group +section "Configuring User Permissions" + +if groups | grep -q libvirt; then + success "User $USER is already in libvirt group" +else + step "Adding user $USER to libvirt group" + if sudo usermod -a -G libvirt "$USER" >> "$LOGFILE" 2>&1; then + success "User added to libvirt group" + warn "You must log out and back in for group membership to take effect" + warn "After logging back in, re-run this script to continue" + exit 0 + else + error "Failed to add user to libvirt group" + fatal "User configuration failed" + fi +fi + +# Verify KVM support +section "Verifying KVM Support" + +if [ -e /dev/kvm ]; then + success "KVM is available" +else + error "KVM is not available" + info "Check if virtualization is enabled in BIOS" + info "Load kvm module: sudo modprobe kvm-intel (or kvm-amd)" + fatal "KVM not available" +fi + +# Check which KVM module is loaded +if lsmod | grep -q kvm_intel; then + info "Using Intel KVM" +elif lsmod | grep -q kvm_amd; then + info "Using AMD KVM" +else + warn "No KVM module detected" + info "Load with: sudo modprobe kvm-intel (or kvm-amd)" +fi + +# Create directory structure +section "Creating Directory Structure" + +DIRS=( + "$PROJECT_ROOT/vm-images" + "$PROJECT_ROOT/test-results" +) + +for dir in "${DIRS[@]}"; do + if [ -d "$dir" ]; then + info "Directory exists: $dir" + else + step "Creating directory: $dir" + if mkdir -p "$dir" 2>> "$LOGFILE"; then + success "Directory created: $dir" + else + error "Failed to create directory: $dir" + fi + fi +done + +# Configure default libvirt network +section "Configuring libvirt Network" + +if virsh net-info default &>/dev/null; then + info "Default network exists" + + if virsh net-info default | grep -q "Active:.*yes"; then + success "Default network is active" + else + step "Starting default network" + if virsh net-start default >> "$LOGFILE" 2>&1; then + success "Default network started" + else + error "Failed to start default network" + fi + fi + + if virsh net-info default | grep -q "Autostart:.*yes"; then + info "Default network autostart is enabled" + else + step "Enabling default network autostart" + if virsh net-autostart default >> "$LOGFILE" 2>&1; then + success "Default network autostart enabled" + else + warn "Failed to enable default network autostart" + fi + fi +else + error "Default network not found" + info "This is unusual - libvirt should create it automatically" +fi + +# Summary +section "Setup Complete" + +success "Testing environment is ready" +info "" +info "Next steps:" +info " 1. Create base VM: ./scripts/testing/create-base-vm.sh" +info " 2. Run a test: ./scripts/testing/run-test.sh" +info "" +info "Log file: $LOGFILE" |