path: root/scripts/testing/lib/validation.sh

#!/bin/bash
# Validation utilities for archsetup testing
# Author: Craig Jennings <craigmartinjennings@gmail.com>
# License: GNU GPLv3
#
# This module provides comprehensive validation checks for archsetup installations.
# It captures pre-install state, runs post-install validations, and attributes
# issues to either archsetup or the base install (archzfs/vanilla Arch).

# Validation counters
VALIDATION_PASSED=0
VALIDATION_FAILED=0
VALIDATION_WARNINGS=0

# Arrays to track issues
declare -a ARCHSETUP_ISSUES
declare -a BASE_INSTALL_ISSUES
declare -a UNKNOWN_ISSUES

# SSH helper (uses globals: VM_IP, ROOT_PASSWORD)
ssh_cmd() {
    sshpass -p "$ROOT_PASSWORD" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
        -o ConnectTimeout=10 "root@$VM_IP" "$@" 2>/dev/null
}

# Validation result helpers
validation_pass() {
    local test_name="$1"
    success "$test_name"
    ((VALIDATION_PASSED++)) || true
}

validation_fail() {
    local test_name="$1"
    local details="${2:-}"
    error "$test_name"
    [ -n "$details" ] && info "  Details: $details"
    ((VALIDATION_FAILED++)) || true
}

validation_warn() {
    local test_name="$1"
    local details="${2:-}"
    warn "$test_name"
    [ -n "$details" ] && info "  Details: $details"
    ((VALIDATION_WARNINGS++)) || true
}

# Attribute an issue to archsetup or base install
attribute_issue() {
    local issue="$1"
    local source="$2"  # "archsetup", "base", or "unknown"

    case "$source" in
        archsetup)
            ARCHSETUP_ISSUES+=("$issue")
            ;;
        base)
            BASE_INSTALL_ISSUES+=("$issue")
            ;;
        *)
            UNKNOWN_ISSUES+=("$issue")
            ;;
    esac
}

#=============================================================================
# PRE-INSTALL LOG CAPTURE
#=============================================================================

capture_pre_install_state() {
    local output_dir="$1"

    section "Capturing Pre-Install State"
    mkdir -p "$output_dir/pre-install"

    step "Capturing system logs before archsetup"

    # Capture journal
    ssh_cmd "journalctl -b --no-pager" > "$output_dir/pre-install/journal.log" 2>&1 || true

    # Capture dmesg
    ssh_cmd "dmesg" > "$output_dir/pre-install/dmesg.log" 2>&1 || true

    # Capture package list
    ssh_cmd "pacman -Q" > "$output_dir/pre-install/packages.txt" 2>&1 || true

    # Capture service status
    ssh_cmd "systemctl list-units --type=service --all" > "$output_dir/pre-install/services.txt" 2>&1 || true

    # Capture failed services
    ssh_cmd "systemctl --failed" > "$output_dir/pre-install/failed-services.txt" 2>&1 || true

    # Capture existing errors in logs
    ssh_cmd "journalctl -b -p err --no-pager" > "$output_dir/pre-install/errors.log" 2>&1 || true

    # Count pre-existing errors
    PRE_INSTALL_ERROR_COUNT=$(wc -l < "$output_dir/pre-install/errors.log" 2>/dev/null || echo 0)

    success "Pre-install state captured ($PRE_INSTALL_ERROR_COUNT pre-existing error lines)"
}

#=============================================================================
# POST-INSTALL LOG CAPTURE
#=============================================================================

capture_post_install_state() {
    local output_dir="$1"

    section "Capturing Post-Install State"
    mkdir -p "$output_dir/post-install"

    step "Capturing system logs after archsetup"

    # Capture journal
    ssh_cmd "journalctl -b --no-pager" > "$output_dir/post-install/journal.log" 2>&1 || true

    # Capture dmesg
    ssh_cmd "dmesg" > "$output_dir/post-install/dmesg.log" 2>&1 || true

    # Capture package list
    ssh_cmd "pacman -Q" > "$output_dir/post-install/packages.txt" 2>&1 || true

    # Capture service status
    ssh_cmd "systemctl list-units --type=service --all" > "$output_dir/post-install/services.txt" 2>&1 || true

    # Capture failed services
    ssh_cmd "systemctl --failed" > "$output_dir/post-install/failed-services.txt" 2>&1 || true

    # Capture all errors
    ssh_cmd "journalctl -b -p err --no-pager" > "$output_dir/post-install/errors.log" 2>&1 || true

    # Capture archsetup log
    ssh_cmd "cat /var/log/archsetup-*.log 2>/dev/null" > "$output_dir/post-install/archsetup.log" 2>&1 || true

    success "Post-install state captured"
}

#=============================================================================
# LOG DIFF ANALYSIS
#=============================================================================

analyze_log_diff() {
    local output_dir="$1"

    section "Analyzing Log Differences"
    mkdir -p "$output_dir/analysis"

    step "Comparing pre and post install errors"

    # Find new errors (in post but not in pre)
    if [ -f "$output_dir/pre-install/errors.log" ] && [ -f "$output_dir/post-install/errors.log" ]; then
        comm -13 <(sort "$output_dir/pre-install/errors.log") <(sort "$output_dir/post-install/errors.log") \
            > "$output_dir/analysis/new-errors.log" 2>/dev/null || true

        NEW_ERROR_COUNT=$(wc -l < "$output_dir/analysis/new-errors.log" 2>/dev/null || echo 0)

        if [ "$NEW_ERROR_COUNT" -gt 0 ]; then
            warn "Found $NEW_ERROR_COUNT new error lines after archsetup"
            # Categorize errors
            categorize_errors "$output_dir/analysis/new-errors.log" "$output_dir/analysis"
        else
            success "No new errors introduced by archsetup"
        fi
    fi

    step "Checking for new failed services"

    # Compare failed services
    if [ -f "$output_dir/pre-install/failed-services.txt" ] && [ -f "$output_dir/post-install/failed-services.txt" ]; then
        local pre_failed post_failed
        pre_failed=$(grep -c "failed" "$output_dir/pre-install/failed-services.txt" 2>/dev/null | tr -d '[:space:]')
        post_failed=$(grep -c "failed" "$output_dir/post-install/failed-services.txt" 2>/dev/null | tr -d '[:space:]')
        # Default to 0 if empty
        pre_failed=${pre_failed:-0}
        post_failed=${post_failed:-0}

        if [ "$post_failed" -gt "$pre_failed" ]; then
            warn "New failed services detected (before: $pre_failed, after: $post_failed)"
            diff "$output_dir/pre-install/failed-services.txt" "$output_dir/post-install/failed-services.txt" \
                > "$output_dir/analysis/failed-services-diff.txt" 2>/dev/null || true
        else
            success "No new service failures"
        fi
    fi

    step "Counting new packages installed"

    if [ -f "$output_dir/pre-install/packages.txt" ] && [ -f "$output_dir/post-install/packages.txt" ]; then
        comm -13 <(sort "$output_dir/pre-install/packages.txt") <(sort "$output_dir/post-install/packages.txt") \
            > "$output_dir/analysis/new-packages.txt" 2>/dev/null || true

        local new_pkg_count=$(wc -l < "$output_dir/analysis/new-packages.txt" 2>/dev/null || echo 0)
        info "Installed $new_pkg_count new packages"
    fi
}

categorize_errors() {
    local error_log="$1"
    local output_dir="$2"

    # Known benign errors/warnings to ignore
    local -a BENIGN_PATTERNS=(
        "SPL:.*module verification failed"
        "ZFS:.*module verification failed"
        "tainting kernel"
        "RAS:.*Correctable Errors"
        "ACPI.*AE_NOT_FOUND"
        "firmware.*regulatory"
        "Invalid user name.*in service file"  # dbus-broker timing during package install
    )

    # Patterns that indicate archsetup issues
    local -a ARCHSETUP_PATTERNS=(
        "archsetup"
        "stow"
        "yay"
        "makepkg"
        "pacman.*error"
    )

    # Filter and categorize
    while IFS= read -r line; do
        local is_benign=false
        local is_archsetup=false

        # Check if benign
        for pattern in "${BENIGN_PATTERNS[@]}"; do
            if echo "$line" | grep -qiE "$pattern"; then
                is_benign=true
                break
            fi
        done

        if $is_benign; then
            echo "$line" >> "$output_dir/benign-errors.log"
            continue
        fi

        # Check if archsetup-related
        for pattern in "${ARCHSETUP_PATTERNS[@]}"; do
            if echo "$line" | grep -qiE "$pattern"; then
                is_archsetup=true
                break
            fi
        done

        if $is_archsetup; then
            echo "$line" >> "$output_dir/archsetup-errors.log"
            attribute_issue "$line" "archsetup"
        else
            echo "$line" >> "$output_dir/base-install-errors.log"
            attribute_issue "$line" "base"
        fi
    done < "$error_log"
}

#=============================================================================
# VALIDATION CHECKS
#=============================================================================

run_all_validations() {
    section "Running Validation Checks"

    # User & Authentication
    validate_user_created
    validate_user_shell
    validate_user_groups

    # Dotfiles
    validate_dotfiles

    # Package Managers
    validate_yay_installed
    validate_pacman_working

    # Window Manager
    validate_suckless_tools

    # Essential Services
    validate_firewall
    validate_dns_config
    validate_avahi
    validate_fail2ban
    validate_networkmanager

    # Developer Tools
    validate_emacs
    validate_git_config
    validate_dev_tools

    # System Configuration
    validate_zfs_config
    validate_boot_config
    validate_autologin_config

    # Archsetup Specific
    validate_archsetup_log
    validate_state_markers
}

#-----------------------------------------------------------------------------
# User & Authentication Validations
#-----------------------------------------------------------------------------

validate_user_created() {
    step "Checking if user 'cjennings' exists"
    if ssh_cmd "id cjennings" &>> "$LOGFILE"; then
        validation_pass "User cjennings exists"
    else
        validation_fail "User cjennings not found"
        attribute_issue "User cjennings not created" "archsetup"
    fi
}

validate_user_shell() {
    step "Checking if ZSH is default shell"
    local shell=$(ssh_cmd "getent passwd cjennings | cut -d: -f7")
    if [ "$shell" = "/bin/zsh" ] || [ "$shell" = "/usr/bin/zsh" ]; then
        validation_pass "ZSH is default shell"
    else
        validation_fail "ZSH not default shell (got: $shell)"
        attribute_issue "ZSH not set as default shell" "archsetup"
    fi
}

validate_user_groups() {
    step "Checking user group memberships"
    # Groups added by archsetup:
    # - wheel (useradd -G wheel)
    # - sys,adm,network,scanner,power,uucp,audio,lp,rfkill,video,storage,optical,users (usermod -aG)
    # - docker (gpasswd -a, added later in developer_workstation)
    local expected_groups="wheel sys adm network scanner power uucp audio lp rfkill video storage optical users docker"
    local missing_groups=""

    for group in $expected_groups; do
        if ! ssh_cmd "groups cjennings" | grep -q "\b$group\b"; then
            missing_groups="$missing_groups $group"
        fi
    done

    if [ -z "$missing_groups" ]; then
        validation_pass "User in all expected groups (15 groups)"
    else
        validation_fail "User missing groups:$missing_groups"
        attribute_issue "User missing groups:$missing_groups" "archsetup"
    fi
}

#-----------------------------------------------------------------------------
# Dotfiles Validations
#-----------------------------------------------------------------------------

validate_dotfiles() {
    step "Checking dotfiles setup"

    # 1. Check if .zshrc is a symlink
    if ! ssh_cmd "test -L /home/cjennings/.zshrc"; then
        validation_fail "Dotfiles not stowed (.zshrc is not a symlink)"
        attribute_issue "Dotfiles stow failed" "archsetup"
        return 1
    fi

    # 2. Check symlink points to correct location
    local target=$(ssh_cmd "readlink /home/cjennings/.zshrc")
    local expected_pattern="code/archsetup/dotfiles/system/.zshrc"

    if ! echo "$target" | grep -q "$expected_pattern"; then
        validation_fail "Dotfiles symlink points to wrong location: $target"
        attribute_issue "Dotfiles symlink incorrect: $target" "archsetup"
        return 1
    fi

    # 3. Check the target file actually exists (not a broken symlink)
    if ! ssh_cmd "test -f /home/cjennings/.zshrc"; then
        validation_fail "Dotfiles symlink is broken (target doesn't exist)"
        ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1
        attribute_issue "Dotfiles symlink broken" "archsetup"
        return 1
    fi

    # 4. Check user can actually read the file (not just root)
    local result=$(ssh_cmd "sudo -u cjennings cat /home/cjennings/.zshrc > /dev/null 2>&1 && echo OK || echo FAIL")
    if [ "$result" != "OK" ]; then
        validation_fail "Dotfiles not readable by user (permission issue)"
        ssh_cmd "ls -la /home/cjennings/.zshrc" >> "$LOGFILE" 2>&1
        attribute_issue "Dotfiles not readable by user" "archsetup"
        return 1
    fi

    validation_pass "Dotfiles configured correctly (symlink to $target, readable by user)"
}

#-----------------------------------------------------------------------------
# Package Manager Validations
#-----------------------------------------------------------------------------

validate_yay_installed() {
    step "Checking if yay (AUR helper) is installed and functional"

    # Check binary exists
    if ! ssh_cmd "which yay" &>> "$LOGFILE"; then
        validation_fail "yay not found"
        attribute_issue "yay not installed" "archsetup"
        return 1
    fi

    # Check yay can query packages (functional test)
    if ssh_cmd "sudo -u cjennings yay -Qi yay" &>> "$LOGFILE"; then
        validation_pass "yay is installed and functional"
    else
        validation_fail "yay binary exists but query failed"
        attribute_issue "yay not functional" "archsetup"
    fi
}

validate_pacman_working() {
    step "Checking if pacman is functional"
    if ssh_cmd "pacman -Qi base" &>> "$LOGFILE"; then
        validation_pass "pacman is functional"
    else
        validation_fail "pacman query failed"
        attribute_issue "pacman not functional" "unknown"
    fi
}

#-----------------------------------------------------------------------------
# Window Manager Validations
#-----------------------------------------------------------------------------

validate_suckless_tools() {
    step "Checking suckless tools (dwm, st, dmenu, slock)"
    local missing=""

    for tool in dwm st dmenu slock; do
        if ! ssh_cmd "test -f /usr/local/bin/$tool"; then
            missing="$missing $tool"
        fi
    done

    if [ -z "$missing" ]; then
        validation_pass "All suckless tools installed (dwm, st, dmenu, slock)"
    else
        validation_fail "Missing suckless tools:$missing"
        attribute_issue "Missing suckless tools:$missing" "archsetup"
    fi
}

#-----------------------------------------------------------------------------
# Essential Services Validations
#-----------------------------------------------------------------------------

validate_firewall() {
    step "Checking if firewall (ufw) is enabled"
    local status=$(ssh_cmd "systemctl is-enabled ufw.service 2>/dev/null || echo disabled")
    if [ "$status" = "enabled" ]; then
        validation_pass "UFW firewall is enabled"
    else
        validation_fail "UFW firewall not enabled"
        attribute_issue "UFW not enabled" "archsetup"
    fi
}

validate_dns_config() {
    step "Checking DNS-over-TLS configuration"
    if ssh_cmd "grep -q 'DNS=.*#' /etc/systemd/resolved.conf 2>/dev/null"; then
        validation_pass "DNS-over-TLS configured"
    else
        validation_warn "DNS-over-TLS may not be configured"
    fi
}

validate_avahi() {
    step "Checking avahi-daemon status"
    local status=$(ssh_cmd "systemctl is-enabled avahi-daemon.service 2>/dev/null || echo disabled")
    if [ "$status" = "enabled" ]; then
        validation_pass "avahi-daemon is enabled"

        # Full-stack mDNS test: ping hostname.local
        local hostname=$(ssh_cmd "hostname")
        if ssh_cmd "ping -c 1 -W 2 ${hostname}.local" &>> "$LOGFILE"; then
            validation_pass "mDNS working (${hostname}.local responds to ping)"
        else
            validation_warn "mDNS ping failed (avahi may need time to propagate)"
        fi
    else
        # This might be OK if avahi was pre-installed
        validation_warn "avahi-daemon not enabled (may have been pre-configured)"
    fi
}

validate_fail2ban() {
    step "Checking fail2ban status"
    local status=$(ssh_cmd "systemctl is-enabled fail2ban.service 2>/dev/null || echo disabled")
    if [ "$status" = "enabled" ]; then
        validation_pass "fail2ban is enabled"
    else
        validation_fail "fail2ban not enabled"
        attribute_issue "fail2ban not enabled" "archsetup"
    fi
}

validate_networkmanager() {
    step "Checking NetworkManager status"
    local status=$(ssh_cmd "systemctl is-enabled NetworkManager.service 2>/dev/null || echo disabled")
    if [ "$status" = "enabled" ]; then
        validation_pass "NetworkManager is enabled"
        # Functional test
        if ssh_cmd "nmcli general status" &>> "$LOGFILE"; then
            validation_pass "NetworkManager is functional"
        else
            validation_warn "NetworkManager enabled but not responding"
        fi
    else
        validation_fail "NetworkManager not enabled"
        attribute_issue "NetworkManager not enabled" "archsetup"
    fi
}

#-----------------------------------------------------------------------------
# Service-Specific Validations
#-----------------------------------------------------------------------------

validate_all_services() {
    section "Service Validations"

    # Core services (always expected)
    validate_service "sshd" "enabled" "active"
    validate_service "systemd-resolved" "enabled" "active"
    validate_service "ufw" "enabled" "active"
    validate_service "fail2ban" "enabled" "active"
    validate_service "NetworkManager" "enabled" "active"
    validate_service "rngd" "enabled" "active"
    validate_service "cronie" "enabled" ""
    validate_service "atd" "enabled" ""

    # Timer services
    validate_service "reflector.timer" "enabled" ""
    validate_service "paccache.timer" "enabled" ""

    # Optional services (warn if missing, don't fail)
    validate_service_optional "avahi-daemon" "enabled"
    validate_service_optional "bluetooth" "enabled"
    validate_service_optional "cups" "enabled"
    validate_service_optional "docker" "enabled"
    validate_service_optional "tailscaled" "enabled"
    validate_service_optional "syncthing@cjennings" "enabled"

    # Filesystem-specific
    validate_zfs_services
    validate_btrfs_services

    # Functional tests
    validate_service_functions
}

validate_service() {
    local service="$1"
    local expected_enabled="$2"  # "enabled" or ""
    local expected_active="$3"   # "active" or ""

    step "Checking $service"

    if [ -n "$expected_enabled" ]; then
        local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled")
        if [ "$enabled" = "enabled" ]; then
            validation_pass "$service is enabled"
        else
            validation_fail "$service not enabled (got: $enabled)"
            attribute_issue "$service not enabled" "archsetup"
            return 1
        fi
    fi

    if [ -n "$expected_active" ]; then
        local active=$(ssh_cmd "systemctl is-active $service 2>/dev/null || echo inactive")
        if [ "$active" = "active" ]; then
            validation_pass "$service is active"
        else
            validation_fail "$service not active (got: $active)"
            attribute_issue "$service not active" "archsetup"
            return 1
        fi
    fi

    return 0
}

validate_service_optional() {
    local service="$1"
    local expected_enabled="$2"

    step "Checking optional service: $service"

    local enabled=$(ssh_cmd "systemctl is-enabled $service 2>/dev/null || echo disabled")
    if [ "$enabled" = "enabled" ]; then
        validation_pass "$service is enabled"
    else
        validation_warn "$service not enabled (optional)"
    fi
}

validate_zfs_services() {
    # Only check if ZFS is installed
    if ! ssh_cmd "which zfs" &>> "$LOGFILE"; then
        return 0
    fi

    step "Checking ZFS-specific services"

    validate_service_optional "sanoid.timer" "enabled"

    # Check for zfs-scrub timer (pool name varies)
    local scrub_enabled=$(ssh_cmd "systemctl list-unit-files 'zfs-scrub*' 2>/dev/null | grep -c enabled || echo 0")
    if [ "$scrub_enabled" -gt 0 ]; then
        validation_pass "ZFS scrub timer enabled"
    else
        validation_warn "ZFS scrub timer not found"
    fi
}

validate_btrfs_services() {
    # Only check if btrfs root
    if ! ssh_cmd "mount | grep 'on / ' | grep -q btrfs"; then
        return 0
    fi

    step "Checking btrfs-specific services"
    validate_service_optional "grub-btrfsd" "enabled"
}

validate_service_functions() {
    section "Service Functional Tests"

    # UFW functional test
    step "Testing UFW functionality"
    local ufw_status
    ufw_status=$(ssh_cmd "ufw status 2>&1 | head -1" | tr -d '[:space:]')
    if echo "$ufw_status" | grep -qi "active"; then
        validation_pass "UFW is active and responding"
    else
        # Check if the service is at least running
        if ssh_cmd "systemctl is-active ufw" &>> "$LOGFILE"; then
            validation_warn "UFW service active but status unclear: $ufw_status"
        else
            validation_fail "UFW not active: $ufw_status"
            attribute_issue "UFW not functioning" "archsetup"
        fi
    fi

    # fail2ban functional test
    step "Testing fail2ban functionality"
    if ssh_cmd "fail2ban-client status" &>> "$LOGFILE"; then
        validation_pass "fail2ban is responding"
    else
        validation_fail "fail2ban not responding"
        attribute_issue "fail2ban not functioning" "archsetup"
    fi

    # DNS resolution test
    step "Testing DNS resolution"
    if ssh_cmd "resolvectl query archlinux.org" &>> "$LOGFILE"; then
        validation_pass "DNS resolution working"
    else
        validation_warn "DNS resolution test failed (may be network issue)"
    fi

    # Docker functional test (if enabled)
    if ssh_cmd "systemctl is-enabled docker" &>> "$LOGFILE"; then
        step "Testing Docker functionality"
        if ssh_cmd "docker info" &>> "$LOGFILE"; then
            validation_pass "Docker is responding"
        else
            validation_warn "Docker enabled but not responding"
        fi
    fi
}

#-----------------------------------------------------------------------------
# Developer Tools Validations
#-----------------------------------------------------------------------------

validate_emacs() {
    step "Checking if Emacs is installed"
    if ssh_cmd "which emacs" &>> "$LOGFILE"; then
        validation_pass "Emacs is installed"

        # Check if config exists
        if ssh_cmd "test -d /home/cjennings/.emacs.d"; then
            validation_pass "Emacs config directory exists"

            # Check user can access the directory
            local result
            result=$(ssh_cmd "sudo -u cjennings ls /home/cjennings/.emacs.d > /dev/null 2>&1 && echo OK || echo FAIL")
            if [ "$result" = "OK" ]; then
                validation_pass "Emacs config readable by user"
            else
                validation_fail "Emacs config not readable by user (permission issue)"
                attribute_issue "Emacs .emacs.d not readable by user" "archsetup"
            fi
        else
            validation_warn "Emacs config directory not found"
        fi
    else
        validation_fail "Emacs not found"
        attribute_issue "Emacs not installed" "archsetup"
    fi
}

validate_git_config() {
    step "Checking git installation"
    if ssh_cmd "which git" &>> "$LOGFILE"; then
        validation_pass "git is installed"
    else
        validation_fail "git not found"
        attribute_issue "git not installed" "archsetup"
    fi
}

validate_dev_tools() {
    step "Checking developer tools"
    local tools="python node npm go rustc"
    local missing=""

    for tool in $tools; do
        if ! ssh_cmd "which $tool" &>> "$LOGFILE"; then
            missing="$missing $tool"
        fi
    done

    if [ -z "$missing" ]; then
        validation_pass "Core dev tools installed"
    else
        validation_warn "Some dev tools missing:$missing"
    fi
}

#-----------------------------------------------------------------------------
# System Configuration Validations
#-----------------------------------------------------------------------------

validate_zfs_config() {
    step "Checking ZFS configuration (if applicable)"
    if ssh_cmd "which zfs" &>> "$LOGFILE"; then
        # ZFS is installed, check for sanoid
        if ssh_cmd "which sanoid" &>> "$LOGFILE"; then
            validation_pass "ZFS with sanoid detected"
        else
            validation_warn "ZFS detected but sanoid not installed"
        fi
    else
        info "ZFS not installed (non-ZFS system)"
    fi
}

validate_boot_config() {
    step "Checking GRUB configuration"
    if ssh_cmd "test -f /boot/grub/grub.cfg" &>> "$LOGFILE"; then
        validation_pass "GRUB config exists"
    else
        validation_warn "GRUB config not found (may use different bootloader)"
    fi
}

validate_autologin_config() {
    step "Checking autologin configuration"
    if ssh_cmd "test -f /etc/systemd/system/getty@tty1.service.d/autologin.conf" &>> "$LOGFILE"; then
        validation_pass "Autologin configured"
    else
        info "Autologin not configured (may be intentional)"
    fi
}

#-----------------------------------------------------------------------------
# Archsetup-Specific Validations
#-----------------------------------------------------------------------------

validate_archsetup_log() {
    step "Checking archsetup log for errors"
    local error_count
    # Use grep -h to suppress filenames, then wc -l to count total matches
    error_count=$(ssh_cmd "grep -h '^Error:' /var/log/archsetup-*.log 2>/dev/null | wc -l" | tr -d '[:space:]')
    error_count=${error_count:-0}

    if [ "$error_count" = "0" ]; then
        validation_pass "No errors in archsetup log"
    else
        validation_fail "Found $error_count errors in archsetup log"
        attribute_issue "Errors in archsetup log: $error_count" "archsetup"
    fi
}

validate_state_markers() {
    step "Checking archsetup state markers"
    local state_count=$(ssh_cmd "ls /var/lib/archsetup/state/ 2>/dev/null | wc -l")

    if [ "$state_count" -ge 12 ]; then
        validation_pass "All 12 installation steps completed"
    else
        validation_warn "Only $state_count/12 steps completed"
    fi
}

#=============================================================================
# ISSUE REPORTING
#=============================================================================

generate_issue_report() {
    local output_dir="$1"
    local archzfs_inbox="$2"

    section "Issue Attribution Report"

    local report_file="$output_dir/issue-report.txt"

    cat > "$report_file" << EOF
========================================
Issue Attribution Report
Generated: $(date +'%Y-%m-%d %H:%M:%S')
========================================

VALIDATION SUMMARY
------------------
Passed:   $VALIDATION_PASSED
Failed:   $VALIDATION_FAILED
Warnings: $VALIDATION_WARNINGS

EOF

    if [ ${#ARCHSETUP_ISSUES[@]} -gt 0 ]; then
        echo "ARCHSETUP ISSUES (${#ARCHSETUP_ISSUES[@]})" >> "$report_file"
        echo "-------------------------------------------" >> "$report_file"
        for issue in "${ARCHSETUP_ISSUES[@]}"; do
            echo "  - $issue" >> "$report_file"
        done
        echo "" >> "$report_file"

        error "Found ${#ARCHSETUP_ISSUES[@]} archsetup issues"
    fi

    if [ ${#BASE_INSTALL_ISSUES[@]} -gt 0 ]; then
        echo "BASE INSTALL ISSUES (${#BASE_INSTALL_ISSUES[@]})" >> "$report_file"
        echo "-------------------------------------------" >> "$report_file"
        for issue in "${BASE_INSTALL_ISSUES[@]}"; do
            echo "  - $issue" >> "$report_file"
        done
        echo "" >> "$report_file"

        warn "Found ${#BASE_INSTALL_ISSUES[@]} base install issues"

        # If archzfs inbox provided, create issue files
        if [ -n "$archzfs_inbox" ] && [ -d "$archzfs_inbox" ]; then
            local issue_file="$archzfs_inbox/$(date +'%Y-%m-%d')-test-issues.txt"
            echo "Base install issues from archsetup test run:" > "$issue_file"
            echo "Date: $(date +'%Y-%m-%d %H:%M:%S')" >> "$issue_file"
            echo "" >> "$issue_file"
            for issue in "${BASE_INSTALL_ISSUES[@]}"; do
                echo "- $issue" >> "$issue_file"
            done
            info "Created archzfs issue file: $issue_file"
        fi
    fi

    if [ ${#UNKNOWN_ISSUES[@]} -gt 0 ]; then
        echo "UNKNOWN/UNATTRIBUTED ISSUES (${#UNKNOWN_ISSUES[@]})" >> "$report_file"
        echo "-------------------------------------------" >> "$report_file"
        for issue in "${UNKNOWN_ISSUES[@]}"; do
            echo "  - $issue" >> "$report_file"
        done
        echo "" >> "$report_file"

        warn "Found ${#UNKNOWN_ISSUES[@]} unattributed issues"
    fi

    if [ ${#ARCHSETUP_ISSUES[@]} -eq 0 ] && [ ${#BASE_INSTALL_ISSUES[@]} -eq 0 ] && [ ${#UNKNOWN_ISSUES[@]} -eq 0 ]; then
        echo "No issues found!" >> "$report_file"
        success "No issues found!"
    fi

    info "Issue report saved: $report_file"
}

#=============================================================================
# MAIN VALIDATION ENTRY POINT
#=============================================================================

run_full_validation() {
    local output_dir="$1"
    local archzfs_inbox="${2:-}"

    run_all_validations
    analyze_log_diff "$output_dir"
    generate_issue_report "$output_dir" "$archzfs_inbox"

    # Return success if no failures
    [ $VALIDATION_FAILED -eq 0 ]
}