From bb3c3ae1b1b1d28fb4253a2fe18d0d53859c143d Mon Sep 17 00:00:00 2001 From: Craig Jennings Date: Tue, 20 Jan 2026 00:09:29 -0600 Subject: feat(testing): add comprehensive validation library for archsetup Add validation.sh library with 25+ automated validation checks: - User creation, shell, and group membership (15 groups) - Dotfiles: symlink validity, target location, and readability - Package managers: yay and pacman functional tests - Suckless tools: dwm, st, dmenu, slock - Services: firewall, DNS-over-TLS, avahi (with mDNS ping test), fail2ban, NetworkManager - Developer tools: emacs, git, python, node, npm, go, rustc - Filesystem-specific: ZFS (sanoid, scrub) and btrfs (grub-btrfsd) - Archsetup-specific: log errors, state markers Also includes: - Pre/post install log capture and diff analysis - Error categorization (benign vs real) - Issue attribution (archsetup vs base install) - archzfs inbox integration for base install issues Co-Authored-By: Claude Opus 4.5 --- scripts/testing/lib/validation.sh | 873 ++++++++++++++++++++++++++++++++++++++ scripts/testing/run-test.sh | 86 ++-- 2 files changed, 906 insertions(+), 53 deletions(-) create mode 100644 scripts/testing/lib/validation.sh diff --git a/scripts/testing/lib/validation.sh b/scripts/testing/lib/validation.sh new file mode 100644 index 0000000..3dc0ce6 --- /dev/null +++ b/scripts/testing/lib/validation.sh @@ -0,0 +1,873 @@ +#!/bin/bash +# Validation utilities for archsetup testing +# Author: Craig Jennings +# License: GNU GPLv3 +# +# This module provides comprehensive validation checks for archsetup installations. +# It captures pre-install state, runs post-install validations, and attributes +# issues to either archsetup or the base install (archzfs/vanilla Arch). + +# Validation counters +VALIDATION_PASSED=0 +VALIDATION_FAILED=0 +VALIDATION_WARNINGS=0 + +# Arrays to track issues +declare -a ARCHSETUP_ISSUES +declare -a BASE_INSTALL_ISSUES +declare -a UNKNOWN_ISSUES + +# SSH helper (uses globals: VM_IP, ROOT_PASSWORD) +ssh_cmd() { + sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 "root@$VM_IP" "$@" 2>/dev/null +} + +# Validation result helpers +validation_pass() { + local test_name="$1" + success "$test_name" + ((VALIDATION_PASSED++)) +} + +validation_fail() { + local test_name="$1" + local details="${2:-}" + error "$test_name" + [ -n "$details" ] && info " Details: $details" + ((VALIDATION_FAILED++)) +} + +validation_warn() { + local test_name="$1" + local details="${2:-}" + warn "$test_name" + [ -n "$details" ] && info " Details: $details" + ((VALIDATION_WARNINGS++)) +} + +# Attribute an issue to archsetup or base install +attribute_issue() { + local issue="$1" + local source="$2" # "archsetup", "base", or "unknown" + + case "$source" in + archsetup) + ARCHSETUP_ISSUES+=("$issue") + ;; + base) + BASE_INSTALL_ISSUES+=("$issue") + ;; + *) + UNKNOWN_ISSUES+=("$issue") + ;; + esac +} + +#============================================================================= +# PRE-INSTALL LOG CAPTURE +#============================================================================= + +capture_pre_install_state() { + local output_dir="$1" + + section "Capturing Pre-Install State" + mkdir -p "$output_dir/pre-install" + + step "Capturing system logs before archsetup" + + # Capture journal + ssh_cmd "journalctl -b --no-pager" > "$output_dir/pre-install/journal.log" 2>&1 || true + + # Capture dmesg + ssh_cmd "dmesg" > "$output_dir/pre-install/dmesg.log" 2>&1 || true + + # Capture package list + ssh_cmd "pacman -Q" > "$output_dir/pre-install/packages.txt" 2>&1 || true + + # Capture service status + ssh_cmd "systemctl list-units --type=service --all" > "$output_dir/pre-install/services.txt" 2>&1 || true + + # Capture failed services + ssh_cmd "systemctl --failed" > "$output_dir/pre-install/failed-services.txt" 2>&1 || true + + # Capture existing errors in logs + ssh_cmd "journalctl -b -p err --no-pager" > "$output_dir/pre-install/errors.log" 2>&1 || true + + # Count pre-existing errors + PRE_INSTALL_ERROR_COUNT=$(wc -l < "$output_dir/pre-install/errors.log" 2>/dev/null || echo 0) + + success "Pre-install state captured ($PRE_INSTALL_ERROR_COUNT pre-existing error lines)" +} + +#============================================================================= +# POST-INSTALL LOG CAPTURE +#============================================================================= + +capture_post_install_state() { + local output_dir="$1" + + section "Capturing Post-Install State" + mkdir -p "$output_dir/post-install" + + step "Capturing system logs after archsetup" + + # Capture journal + ssh_cmd "journalctl -b --no-pager" > "$output_dir/post-install/journal.log" 2>&1 || true + + # Capture dmesg + ssh_cmd "dmesg" > "$output_dir/post-install/dmesg.log" 2>&1 || true + + # Capture package list + ssh_cmd "pacman -Q" > "$output_dir/post-install/packages.txt" 2>&1 || true + + # Capture service status + ssh_cmd "systemctl list-units --type=service --all" > "$output_dir/post-install/services.txt" 2>&1 || true + + # Capture failed services + ssh_cmd "systemctl --failed" > "$output_dir/post-install/failed-services.txt" 2>&1 || true + + # Capture all errors + ssh_cmd "journalctl -b -p err --no-pager" > "$output_dir/post-install/errors.log" 2>&1 || true + + # Capture archsetup log + ssh_cmd "cat /var/log/archsetup-*.log 2>/dev/null" > "$output_dir/post-install/archsetup.log" 2>&1 || true + + success "Post-install state captured" +} + +#============================================================================= +# LOG DIFF ANALYSIS +#============================================================================= + +analyze_log_diff() { + local output_dir="$1" + + section "Analyzing Log Differences" + mkdir -p "$output_dir/analysis" + + step "Comparing pre and post install errors" + + # Find new errors (in post but not in pre) + if [ -f "$output_dir/pre-install/errors.log" ] && [ -f "$output_dir/post-install/errors.log" ]; then + comm -13 <(sort "$output_dir/pre-install/errors.log") <(sort "$output_dir/post-install/errors.log") \ + > "$output_dir/analysis/new-errors.log" 2>/dev/null || true + + NEW_ERROR_COUNT=$(wc -l < "$output_dir/analysis/new-errors.log" 2>/dev/null || echo 0) + + if [ "$NEW_ERROR_COUNT" -gt 0 ]; then + warn "Found $NEW_ERROR_COUNT new error lines after archsetup" + # Categorize errors + categorize_errors "$output_dir/analysis/new-errors.log" "$output_dir/analysis" + else + success "No new errors introduced by archsetup" + fi + fi + + step "Checking for new failed services" + + # Compare failed services + if [ -f "$output_dir/pre-install/failed-services.txt" ] && [ -f "$output_dir/post-install/failed-services.txt" ]; then + local pre_failed=$(grep -c "failed" "$output_dir/pre-install/failed-services.txt" 2>/dev/null || echo 0) + local post_failed=$(grep -c "failed" "$output_dir/post-install/failed-services.txt" 2>/dev/null || echo 0) + + if [ "$post_failed" -gt "$pre_failed" ]; then + warn "New failed services detected (before: $pre_failed, after: $post_failed)" + diff "$output_dir/pre-install/failed-services.txt" "$output_dir/post-install/failed-services.txt" \ + > "$output_dir/analysis/failed-services-diff.txt" 2>/dev/null || true + else + success "No new service failures" + fi + fi + + step "Counting new packages installed" + + if [ -f "$output_dir/pre-install/packages.txt" ] && [ -f "$output_dir/post-install/packages.txt" ]; then + comm -13 <(sort "$output_dir/pre-install/packages.txt") <(sort "$output_dir/post-install/packages.txt") \ + > "$output_dir/analysis/new-packages.txt" 2>/dev/null || true + + local new_pkg_count=$(wc -l < "$output_dir/analysis/new-packages.txt" 2>/dev/null || echo 0) + info "Installed $new_pkg_count new packages" + fi +} + +categorize_errors() { + local error_log="$1" + local output_dir="$2" + + # Known benign errors/warnings to ignore + local -a BENIGN_PATTERNS=( + "SPL:.*module verification failed" + "ZFS:.*module verification failed" + "tainting kernel" + "RAS:.*Correctable Errors" + "ACPI.*AE_NOT_FOUND" + "firmware.*regulatory" + ) + + # Patterns that indicate archsetup issues + local -a ARCHSETUP_PATTERNS=( + "archsetup" + "stow" + "yay" + "makepkg" + "pacman.*error" + ) + + # Filter and categorize + while IFS= read -r line; do + local is_benign=false + local is_archsetup=false + + # Check if benign + for pattern in "${BENIGN_PATTERNS[@]}"; do + if echo "$line" | grep -qiE "$pattern"; then + is_benign=true + break + fi + done + + if $is_benign; then + echo "$line" >> "$output_dir/benign-errors.log" + continue + fi + + # Check if archsetup-related + for pattern in "${ARCHSETUP_PATTERNS[@]}"; do + if echo "$line" | grep -qiE "$pattern"; then + is_archsetup=true + break + fi + done + + if $is_archsetup; then + echo "$line" >> "$output_dir/archsetup-errors.log" + attribute_issue "$line" "archsetup" + else + echo "$line" >> "$output_dir/base-install-errors.log" + attribute_issue "$line" "base" + fi + done < "$error_log" +} + +#============================================================================= +# VALIDATION CHECKS +#============================================================================= + +run_all_validations() { + section "Running Validation Checks" + + # User & Authentication + validate_user_created + validate_user_shell + validate_user_groups + + # Dotfiles + validate_dotfiles + + # Package Managers + validate_yay_installed + validate_pacman_working + + # Window Manager + validate_suckless_tools + + # Essential Services + validate_firewall + validate_dns_config + validate_avahi + validate_fail2ban + validate_networkmanager + + # Developer Tools + validate_emacs + validate_git_config + validate_dev_tools + + # System Configuration + validate_zfs_config + validate_boot_config + validate_autologin_config + + # Archsetup Specific + validate_archsetup_log + validate_state_markers +} + +#----------------------------------------------------------------------------- +# User & Authentication Validations +#----------------------------------------------------------------------------- + +validate_user_created() { + step "Checking if user 'cjennings' exists" + if ssh_cmd "id cjennings" &>> "$LOGFILE"; then + validation_pass "User cjennings exists" + else + validation_fail "User cjennings not found" + attribute_issue "User cjennings not created" "archsetup" + fi +} + +validate_user_shell() { + step "Checking if ZSH is default shell" + local shell=$(ssh_cmd "getent passwd cjennings | cut -d: -f7") + if [ "$shell" = "/bin/zsh" ] || [ "$shell" = "/usr/bin/zsh" ]; then + validation_pass "ZSH is default shell" + else + validation_fail "ZSH not default shell (got: $shell)" + attribute_issue "ZSH not set as default shell" "archsetup" + fi +} + +validate_user_groups() { + step "Checking user group memberships" + # Groups added by archsetup: + # - wheel (useradd -G wheel) + # - sys,adm,network,scanner,power,uucp,audio,lp,rfkill,video,storage,optical,users (usermod -aG) + # - docker (gpasswd -a, added later in developer_workstation) + local expected_groups="wheel sys adm network scanner power uucp audio lp rfkill video storage optical users docker" + local missing_groups="" + + for group in $expected_groups; do + if ! ssh_cmd "groups cjennings" | grep -q "\b$group\b"; then + missing_groups="$missing_groups $group" + fi + done + + if [ -z "$missing_groups" ]; then + validation_pass "User in all expected groups (15 groups)" + else + validation_fail "User missing groups:$missing_groups" + attribute_issue "User missing groups:$missing_groups" "archsetup" + fi +} + +#----------------------------------------------------------------------------- +# Dotfiles Validations +#----------------------------------------------------------------------------- + +validate_dotfiles() { + step "Checking dotfiles setup" + + # 1. Check if .zshrc is a symlink + if ! ssh_cmd "test -L /home/cjennings/.zshrc"; then + validation_fail "Dotfiles not stowed (.zshrc is not a symlink)" + attribute_issue "Dotfiles stow failed" "archsetup" + return 1 + fi + + # 2. Check symlink points to correct location + local target=$(ssh_cmd "readlink /home/cjennings/.zshrc") + local expected_pattern="code/archsetup/dotfiles/system/.zshrc" + + if ! echo "$target" | grep -q "$expected_pattern"; then + validation_fail "Dotfiles symlink points to wrong location: $target" + attribute_issue "Dotfiles symlink incorrect: $target" "archsetup" + return 1 + fi + + # 3. Check the target file actually exists (not a broken symlink) + if ! ssh_cmd "test -f /home/cjennings/.zshrc"; then + validation_fail "Dotfiles symlink is broken (target doesn't exist)" + ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 + attribute_issue "Dotfiles symlink broken" "archsetup" + return 1 + fi + + # 4. Check user can actually read the file (not just root) + local result=$(ssh_cmd "sudo -u cjennings cat /home/cjennings/.zshrc > /dev/null 2>&1 && echo OK || echo FAIL") + if [ "$result" != "OK" ]; then + validation_fail "Dotfiles not readable by user (permission issue)" + ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 + attribute_issue "Dotfiles not readable by user" "archsetup" + return 1 + fi + + validation_pass "Dotfiles configured correctly (symlink to $target, readable by user)" +} + +#----------------------------------------------------------------------------- +# Package Manager Validations +#----------------------------------------------------------------------------- + +validate_yay_installed() { + step "Checking if yay (AUR helper) is installed and functional" + + # Check binary exists + if ! ssh_cmd "which yay" &>> "$LOGFILE"; then + validation_fail "yay not found" + attribute_issue "yay not installed" "archsetup" + return 1 + fi + + # Check yay can query packages (functional test) + if ssh_cmd "sudo -u cjennings yay -Qi yay" &>> "$LOGFILE"; then + validation_pass "yay is installed and functional" + else + validation_fail "yay binary exists but query failed" + attribute_issue "yay not functional" "archsetup" + fi +} + +validate_pacman_working() { + step "Checking if pacman is functional" + if ssh_cmd "pacman -Qi base" &>> "$LOGFILE"; then + validation_pass "pacman is functional" + else + validation_fail "pacman query failed" + attribute_issue "pacman not functional" "unknown" + fi +} + +#----------------------------------------------------------------------------- +# Window Manager Validations +#----------------------------------------------------------------------------- + +validate_suckless_tools() { + step "Checking suckless tools (dwm, st, dmenu, slock)" + local missing="" + + for tool in dwm st dmenu slock; do + if ! ssh_cmd "test -f /usr/local/bin/$tool"; then + missing="$missing $tool" + fi + done + + if [ -z "$missing" ]; then + validation_pass "All suckless tools installed (dwm, st, dmenu, slock)" + else + validation_fail "Missing suckless tools:$missing" + attribute_issue "Missing suckless tools:$missing" "archsetup" + fi +} + +#----------------------------------------------------------------------------- +# Essential Services Validations +#----------------------------------------------------------------------------- + +validate_firewall() { + step "Checking if firewall (ufw) is enabled" + local status=$(ssh_cmd "systemctl is-enabled ufw.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "UFW firewall is enabled" + else + validation_fail "UFW firewall not enabled" + attribute_issue "UFW not enabled" "archsetup" + fi +} + +validate_dns_config() { + step "Checking DNS-over-TLS configuration" + if ssh_cmd "grep -q 'DNS=.*#' /etc/systemd/resolved.conf 2>/dev/null"; then + validation_pass "DNS-over-TLS configured" + else + validation_warn "DNS-over-TLS may not be configured" + fi +} + +validate_avahi() { + step "Checking avahi-daemon status" + local status=$(ssh_cmd "systemctl is-enabled avahi-daemon.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "avahi-daemon is enabled" + + # Full-stack mDNS test: ping hostname.local + local hostname=$(ssh_cmd "hostname") + if ssh_cmd "ping -c 1 -W 2 ${hostname}.local" &>> "$LOGFILE"; then + validation_pass "mDNS working (${hostname}.local responds to ping)" + else + validation_warn "mDNS ping failed (avahi may need time to propagate)" + fi + else + # This might be OK if avahi was pre-installed + validation_warn "avahi-daemon not enabled (may have been pre-configured)" + fi +} + +validate_fail2ban() { + step "Checking fail2ban status" + local status=$(ssh_cmd "systemctl is-enabled fail2ban.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "fail2ban is enabled" + else + validation_fail "fail2ban not enabled" + attribute_issue "fail2ban not enabled" "archsetup" + fi +} + +validate_networkmanager() { + step "Checking NetworkManager status" + local status=$(ssh_cmd "systemctl is-enabled NetworkManager.service 2>/dev/null || echo disabled") + if [ "$status" = "enabled" ]; then + validation_pass "NetworkManager is enabled" + # Functional test + if ssh_cmd "nmcli general status" &>> "$LOGFILE"; then + validation_pass "NetworkManager is functional" + else + validation_warn "NetworkManager enabled but not responding" + fi + else + validation_fail "NetworkManager not enabled" + attribute_issue "NetworkManager not enabled" "archsetup" + fi +} + +#----------------------------------------------------------------------------- +# Service-Specific Validations +#----------------------------------------------------------------------------- + +validate_all_services() { + section "Service Validations" + + # Core services (always expected) + validate_service "sshd" "enabled" "active" + validate_service "systemd-resolved" "enabled" "active" + validate_service "ufw" "enabled" "active" + validate_service "fail2ban" "enabled" "active" + validate_service "NetworkManager" "enabled" "active" + validate_service "rngd" "enabled" "active" + validate_service "cronie" "enabled" "" + validate_service "atd" "enabled" "" + + # Timer services + validate_service "reflector.timer" "enabled" "" + validate_service "paccache.timer" "enabled" "" + + # Optional services (warn if missing, don't fail) + validate_service_optional "avahi-daemon" "enabled" + validate_service_optional "bluetooth" "enabled" + validate_service_optional "cups" "enabled" + validate_service_optional "docker" "enabled" + validate_service_optional "tailscaled" "enabled" + validate_service_optional "syncthing@cjennings" "enabled" + + # Filesystem-specific + validate_zfs_services + validate_btrfs_services + + # Functional tests + validate_service_functions +} + +validate_service() { + local service="$1" + local expected_enabled="$2" # "enabled" or "" + local expected_active="$3" # "active" or "" + + step "Checking $service" + + if [ -n "$expected_enabled" ]; then + local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled") + if [ "$enabled" = "enabled" ]; then + validation_pass "$service is enabled" + else + validation_fail "$service not enabled (got: $enabled)" + attribute_issue "$service not enabled" "archsetup" + return 1 + fi + fi + + if [ -n "$expected_active" ]; then + local active=$(ssh_cmd "systemctl is-active $service 2>/dev/null || echo inactive") + if [ "$active" = "active" ]; then + validation_pass "$service is active" + else + validation_fail "$service not active (got: $active)" + attribute_issue "$service not active" "archsetup" + return 1 + fi + fi + + return 0 +} + +validate_service_optional() { + local service="$1" + local expected_enabled="$2" + + step "Checking optional service: $service" + + local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled") + if [ "$enabled" = "enabled" ]; then + validation_pass "$service is enabled" + else + validation_warn "$service not enabled (optional)" + fi +} + +validate_zfs_services() { + # Only check if ZFS is installed + if ! ssh_cmd "which zfs" &>> "$LOGFILE"; then + return 0 + fi + + step "Checking ZFS-specific services" + + validate_service_optional "sanoid.timer" "enabled" + + # Check for zfs-scrub timer (pool name varies) + local scrub_enabled=$(ssh_cmd "systemctl list-unit-files 'zfs-scrub*' 2>/dev/null | grep -c enabled || echo 0") + if [ "$scrub_enabled" -gt 0 ]; then + validation_pass "ZFS scrub timer enabled" + else + validation_warn "ZFS scrub timer not found" + fi +} + +validate_btrfs_services() { + # Only check if btrfs root + if ! ssh_cmd "mount | grep 'on / ' | grep -q btrfs"; then + return 0 + fi + + step "Checking btrfs-specific services" + validate_service_optional "grub-btrfsd" "enabled" +} + +validate_service_functions() { + section "Service Functional Tests" + + # UFW functional test + step "Testing UFW functionality" + local ufw_status=$(ssh_cmd "ufw status 2>/dev/null | head -1") + if echo "$ufw_status" | grep -q "active"; then + validation_pass "UFW is active and responding" + else + validation_fail "UFW not active: $ufw_status" + attribute_issue "UFW not functioning" "archsetup" + fi + + # fail2ban functional test + step "Testing fail2ban functionality" + if ssh_cmd "fail2ban-client status" &>> "$LOGFILE"; then + validation_pass "fail2ban is responding" + else + validation_fail "fail2ban not responding" + attribute_issue "fail2ban not functioning" "archsetup" + fi + + # DNS resolution test + step "Testing DNS resolution" + if ssh_cmd "resolvectl query archlinux.org" &>> "$LOGFILE"; then + validation_pass "DNS resolution working" + else + validation_warn "DNS resolution test failed (may be network issue)" + fi + + # Docker functional test (if enabled) + if ssh_cmd "systemctl is-enabled docker" &>> "$LOGFILE"; then + step "Testing Docker functionality" + if ssh_cmd "docker info" &>> "$LOGFILE"; then + validation_pass "Docker is responding" + else + validation_warn "Docker enabled but not responding" + fi + fi +} + +#----------------------------------------------------------------------------- +# Developer Tools Validations +#----------------------------------------------------------------------------- + +validate_emacs() { + step "Checking if Emacs is installed" + if ssh_cmd "which emacs" &>> "$LOGFILE"; then + validation_pass "Emacs is installed" + + # Check if config exists + if ssh_cmd "test -d /home/cjennings/.emacs.d"; then + validation_pass "Emacs config directory exists" + else + validation_warn "Emacs config directory not found" + fi + else + validation_fail "Emacs not found" + attribute_issue "Emacs not installed" "archsetup" + fi +} + +validate_git_config() { + step "Checking git installation" + if ssh_cmd "which git" &>> "$LOGFILE"; then + validation_pass "git is installed" + else + validation_fail "git not found" + attribute_issue "git not installed" "archsetup" + fi +} + +validate_dev_tools() { + step "Checking developer tools" + local tools="python node npm go rustc" + local missing="" + + for tool in $tools; do + if ! ssh_cmd "which $tool" &>> "$LOGFILE"; then + missing="$missing $tool" + fi + done + + if [ -z "$missing" ]; then + validation_pass "Core dev tools installed" + else + validation_warn "Some dev tools missing:$missing" + fi +} + +#----------------------------------------------------------------------------- +# System Configuration Validations +#----------------------------------------------------------------------------- + +validate_zfs_config() { + step "Checking ZFS configuration (if applicable)" + if ssh_cmd "which zfs" &>> "$LOGFILE"; then + # ZFS is installed, check for sanoid + if ssh_cmd "which sanoid" &>> "$LOGFILE"; then + validation_pass "ZFS with sanoid detected" + else + validation_warn "ZFS detected but sanoid not installed" + fi + else + info "ZFS not installed (non-ZFS system)" + fi +} + +validate_boot_config() { + step "Checking GRUB configuration" + if ssh_cmd "test -f /boot/grub/grub.cfg" &>> "$LOGFILE"; then + validation_pass "GRUB config exists" + else + validation_warn "GRUB config not found (may use different bootloader)" + fi +} + +validate_autologin_config() { + step "Checking autologin configuration" + if ssh_cmd "test -f /etc/systemd/system/getty@tty1.service.d/autologin.conf" &>> "$LOGFILE"; then + validation_pass "Autologin configured" + else + info "Autologin not configured (may be intentional)" + fi +} + +#----------------------------------------------------------------------------- +# Archsetup-Specific Validations +#----------------------------------------------------------------------------- + +validate_archsetup_log() { + step "Checking archsetup log for errors" + local error_count=$(ssh_cmd "grep -c '^Error:' /var/log/archsetup-*.log 2>/dev/null || echo 0") + + if [ "$error_count" = "0" ]; then + validation_pass "No errors in archsetup log" + else + validation_fail "Found $error_count errors in archsetup log" + attribute_issue "Errors in archsetup log: $error_count" "archsetup" + fi +} + +validate_state_markers() { + step "Checking archsetup state markers" + local state_count=$(ssh_cmd "ls /var/lib/archsetup/state/ 2>/dev/null | wc -l") + + if [ "$state_count" -ge 12 ]; then + validation_pass "All 12 installation steps completed" + else + validation_warn "Only $state_count/12 steps completed" + fi +} + +#============================================================================= +# ISSUE REPORTING +#============================================================================= + +generate_issue_report() { + local output_dir="$1" + local archzfs_inbox="$2" + + section "Issue Attribution Report" + + local report_file="$output_dir/issue-report.txt" + + cat > "$report_file" << EOF +======================================== +Issue Attribution Report +Generated: $(date +'%Y-%m-%d %H:%M:%S') +======================================== + +VALIDATION SUMMARY +------------------ +Passed: $VALIDATION_PASSED +Failed: $VALIDATION_FAILED +Warnings: $VALIDATION_WARNINGS + +EOF + + if [ ${#ARCHSETUP_ISSUES[@]} -gt 0 ]; then + echo "ARCHSETUP ISSUES (${#ARCHSETUP_ISSUES[@]})" >> "$report_file" + echo "-------------------------------------------" >> "$report_file" + for issue in "${ARCHSETUP_ISSUES[@]}"; do + echo " - $issue" >> "$report_file" + done + echo "" >> "$report_file" + + error "Found ${#ARCHSETUP_ISSUES[@]} archsetup issues" + fi + + if [ ${#BASE_INSTALL_ISSUES[@]} -gt 0 ]; then + echo "BASE INSTALL ISSUES (${#BASE_INSTALL_ISSUES[@]})" >> "$report_file" + echo "-------------------------------------------" >> "$report_file" + for issue in "${BASE_INSTALL_ISSUES[@]}"; do + echo " - $issue" >> "$report_file" + done + echo "" >> "$report_file" + + warn "Found ${#BASE_INSTALL_ISSUES[@]} base install issues" + + # If archzfs inbox provided, create issue files + if [ -n "$archzfs_inbox" ] && [ -d "$archzfs_inbox" ]; then + local issue_file="$archzfs_inbox/$(date +'%Y-%m-%d')-test-issues.txt" + echo "Base install issues from archsetup test run:" > "$issue_file" + echo "Date: $(date +'%Y-%m-%d %H:%M:%S')" >> "$issue_file" + echo "" >> "$issue_file" + for issue in "${BASE_INSTALL_ISSUES[@]}"; do + echo "- $issue" >> "$issue_file" + done + info "Created archzfs issue file: $issue_file" + fi + fi + + if [ ${#UNKNOWN_ISSUES[@]} -gt 0 ]; then + echo "UNKNOWN/UNATTRIBUTED ISSUES (${#UNKNOWN_ISSUES[@]})" >> "$report_file" + echo "-------------------------------------------" >> "$report_file" + for issue in "${UNKNOWN_ISSUES[@]}"; do + echo " - $issue" >> "$report_file" + done + echo "" >> "$report_file" + + warn "Found ${#UNKNOWN_ISSUES[@]} unattributed issues" + fi + + if [ ${#ARCHSETUP_ISSUES[@]} -eq 0 ] && [ ${#BASE_INSTALL_ISSUES[@]} -eq 0 ] && [ ${#UNKNOWN_ISSUES[@]} -eq 0 ]; then + echo "No issues found!" >> "$report_file" + success "No issues found!" + fi + + info "Issue report saved: $report_file" +} + +#============================================================================= +# MAIN VALIDATION ENTRY POINT +#============================================================================= + +run_full_validation() { + local output_dir="$1" + local archzfs_inbox="${2:-}" + + run_all_validations + analyze_log_diff "$output_dir" + generate_issue_report "$output_dir" "$archzfs_inbox" + + # Return success if no failures + [ $VALIDATION_FAILED -eq 0 ] +} diff --git a/scripts/testing/run-test.sh b/scripts/testing/run-test.sh index e73f29c..e06d177 100755 --- a/scripts/testing/run-test.sh +++ b/scripts/testing/run-test.sh @@ -22,6 +22,7 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "$SCRIPT_DIR/lib/logging.sh" source "$SCRIPT_DIR/lib/vm-utils.sh" source "$SCRIPT_DIR/lib/network-diagnostics.sh" +source "$SCRIPT_DIR/lib/validation.sh" # Parse arguments KEEP_VM=false @@ -63,6 +64,7 @@ TIMESTAMP=$(date +'%Y%m%d-%H%M%S') VM_NAME="archsetup-base" TEST_RESULTS_DIR="$PROJECT_ROOT/test-results/$TIMESTAMP" ROOT_PASSWORD="archsetup" +ARCHZFS_INBOX="$HOME/code/archzfs/inbox" # Initialize logging mkdir -p "$TEST_RESULTS_DIR" @@ -155,6 +157,9 @@ if ! run_network_diagnostics "$VM_IP"; then fatal "Network diagnostics failed - aborting test" fi +# Capture pre-install state for comparison +capture_pre_install_state "$TEST_RESULTS_DIR" + # Transfer files to VM (simulating git clone) section "Simulating Git Clone" @@ -271,64 +276,25 @@ sshpass -p "$ROOT_PASSWORD" scp -o StrictHostKeyChecking=no -o UserKnownHostsFil "root@$VM_IP:/root/.local/src/archsetup-*.txt" "$TEST_RESULTS_DIR/" 2>> "$LOGFILE" || \ warn "Could not copy package lists" -# Run validation -section "Validating Installation" +# Capture post-install state +capture_post_install_state "$TEST_RESULTS_DIR" -VALIDATION_PASSED=true +# Run comprehensive validation +# This uses the validation.sh library for all checks +run_all_validations +validate_all_services -# Check if user was created -step "Checking if user 'cjennings' was created" -if sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$VM_IP" "id cjennings" &>> "$LOGFILE"; then - success "User cjennings exists" -else - error "User cjennings not found" - VALIDATION_PASSED=false -fi +# Analyze log differences (pre vs post install) +analyze_log_diff "$TEST_RESULTS_DIR" -# Check if dotfiles were stowed -step "Checking if dotfiles are stowed" -if sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$VM_IP" "test -L /home/cjennings/.profile" &>> "$LOGFILE"; then - success "Dotfiles appear to be stowed" -else - warn "Dotfiles may not be properly stowed" - VALIDATION_PASSED=false -fi +# Generate issue attribution report +# If base install issues found and archzfs inbox exists, create issue file +generate_issue_report "$TEST_RESULTS_DIR" "$ARCHZFS_INBOX" -# Check if dotfile symlinks are accessible by user (not just root) -# This catches the bug where stow creates symlinks pointing to /root/archsetup or /tmp -# Use sudo -u to avoid starting a full login shell which might hang -step "Checking if dotfiles are readable by user cjennings" -DOTFILE_CHECK=$(sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$VM_IP" "sudo -u cjennings cat /home/cjennings/.zshrc > /dev/null 2>&1 && echo OK || echo FAIL" 2>/dev/null) -if [ "$DOTFILE_CHECK" = "OK" ]; then - success "Dotfiles are readable by user cjennings" +# Set validation result based on failure count +if [ $VALIDATION_FAILED -eq 0 ]; then + VALIDATION_PASSED=true else - error "Dotfiles are NOT readable by user cjennings (symlinks may point to inaccessible location)" - # Show where the symlink points for debugging - sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$VM_IP" "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1 - VALIDATION_PASSED=false -fi - -# Check if yay is installed -step "Checking if yay (AUR helper) is installed" -if sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$VM_IP" "/usr/bin/which yay" &>> "$LOGFILE"; then - success "yay is installed" -else - error "yay not found" - VALIDATION_PASSED=false -fi - -# Check if DWM was built -step "Checking if DWM is installed" -if sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - "root@$VM_IP" "test -f /usr/local/bin/dwm" &>> "$LOGFILE"; then - success "DWM is installed" -else - error "DWM not found" VALIDATION_PASSED=false fi @@ -354,10 +320,24 @@ Results: ArchSetup Exit Code: $ARCHSETUP_EXIT_CODE Validation: $(if $VALIDATION_PASSED; then echo "PASSED"; else echo "FAILED"; fi) +Validation Summary: + Passed: $VALIDATION_PASSED + Failed: $VALIDATION_FAILED + Warnings: $VALIDATION_WARNINGS + +Issue Attribution: + ArchSetup Issues: ${#ARCHSETUP_ISSUES[@]} + Base Install Issues: ${#BASE_INSTALL_ISSUES[@]} + Unknown Issues: ${#UNKNOWN_ISSUES[@]} + Artifacts: Log file: $LOGFILE Report: $REPORT_FILE Results: $TEST_RESULTS_DIR/ + Issue Report: $TEST_RESULTS_DIR/issue-report.txt + Pre-install logs: $TEST_RESULTS_DIR/pre-install/ + Post-install logs: $TEST_RESULTS_DIR/post-install/ + Analysis: $TEST_RESULTS_DIR/analysis/ EOFREPORT -- cgit v1.2.3