diff options
27 files changed, 3485 insertions, 39 deletions
@@ -5,7 +5,7 @@ # (https://git.cjennings.net/dotfiles.git). Run them from there: # cd ~/.dotfiles && make stow|restow|reset|unstow|import <de> -.PHONY: help deps test-unit test test-keep test-vm-base package-diff +.PHONY: help deps test-unit test test-keep test-vm-base test-maint package-diff # Filesystem profile for the VM harness: btrfs (default) or zfs. Selects the # base image the scripts build/use; exported so create-base-vm.sh + run-test.sh @@ -28,6 +28,7 @@ help: @echo " test Run full VM test suite (creates base VM if needed)" @echo " test-keep Run test and keep VM running for manual testing" @echo " test-vm-base Create base VM only (runs archangel)" + @echo " test-maint Run maint remedy scenarios in the VM (break/fix/assert)" @echo " package-diff Compare archsetup's declared packages vs this system" @echo "" @echo "Filesystem profile (test, test-keep, test-vm-base):" @@ -81,6 +82,18 @@ test-keep: fi @bash scripts/testing/run-test.sh --keep +# Maintenance-console remedy scenarios: break the VM, run the real +# `maint fix`, assert the post-state. Grouped batches over the clean-install +# snapshot; the base image is restored afterwards. `--list` variant to +# preview: bash scripts/testing/run-maint-scenarios.sh --list +test-maint: + @if [ ! -f $(BASE_IMAGE) ] || \ + ! qemu-img snapshot -l $(BASE_IMAGE) 2>/dev/null | grep -q "clean-install"; then \ + echo "Base VM not found or missing snapshot, creating ($(FS_PROFILE))..."; \ + bash scripts/testing/create-base-vm.sh; \ + fi + @bash scripts/testing/run-maint-scenarios.sh + # Compare the packages archsetup declares against what's installed here. # Shows declared-but-missing and installed-but-undeclared (AUR vs official). package-diff: @@ -732,7 +732,12 @@ retry_install() { # Pacman Install pacman_install() { - retry_install "$1" "pacman" "pacman --noconfirm --needed -S \"$1\"" + retry_install "$1" "pacman" "pacman --noconfirm --needed -S \"$1\"" || return $? + # --needed skips a package already present as a dependency and leaves + # its install reason alone; the package can then surface as an orphan + # later and get swept away (expac/lm_sensors nearly went this way, + # 2026-07-08). Every declared package is wanted explicitly. + pacman -D --asexplicit "$1" >> "$logfile" 2>&1 || true } # AUR Install @@ -1210,6 +1215,7 @@ user_customizations() { refresh_desktop_caches configure_dconf_defaults finalize_dotfiles + install_maintenance_config create_user_directories } @@ -1358,6 +1364,45 @@ finalize_dotfiles() { } +install_maintenance_config() { + + display "subtitle" "Maintenance Console" + + install_maintenance_thresholds + if [[ "$desktop_env" == "hyprland" ]]; then + enable_maint_timers + fi +} + +install_maintenance_thresholds() { + # Shipped severity thresholds read by the maint console and the + # system-health-check workflow. This file is the shipped layer — a re-run + # refreshes it. User curation and overrides live in ~/.config/maint/, + # which archsetup never touches, so a reinstall can't eat curation. + action="installing maintenance thresholds" && display "task" "$action" + local src="${user_archsetup_dir:-/home/$username/code/archsetup}/configs/maintenance-thresholds.toml" + (install -d "/home/$username/.config/archsetup" && \ + install -m 0644 "$src" "/home/$username/.config/archsetup/maintenance-thresholds.toml" && \ + chown -R "$username:$username" "/home/$username/.config/archsetup") \ + >> "$logfile" 2>&1 || error_warn "$action" "$?" + +} + +enable_maint_timers() { + # The hyprland tier stows maint-scan.timer + maint-net-scan.timer as user + # units under ~/.config/systemd/user/. systemctl --user can't run during + # install (no user session bus), so create the enablement symlinks + # directly — same idiom as the syncthing user service. + action="enabling maint scan timers" && display "task" "$action" + local unit_dir="/home/$username/.config/systemd/user" + (mkdir -p "$unit_dir/timers.target.wants" && \ + ln -sf "$unit_dir/maint-scan.timer" "$unit_dir/timers.target.wants/maint-scan.timer" && \ + ln -sf "$unit_dir/maint-net-scan.timer" "$unit_dir/timers.target.wants/maint-net-scan.timer" && \ + chown -R "$username:$username" "/home/$username/.config/systemd") \ + >> "$logfile" 2>&1 || error_warn "$action" "$?" + +} + create_user_directories() { action="creating common directories" && display "task" "$action" # Create default directories and grant permissions @@ -1673,6 +1718,7 @@ configure_package_cache() { display "subtitle" "Package Repository Cache Maintenance" pacman_install pacman-contrib pacman_install arch-audit # CVE report from Arch Security Team data (maintenance console) + pacman_install expac # package metadata queries (maintenance console) run_task "enabling the package cache cleanup timer" systemctl enable --now paccache.timer action="configuring paccache to keep 3 versions" && display "task" "$action" @@ -2744,6 +2790,8 @@ supplemental_software() { pacman_install bind # DNS utilities (dig, host, nslookup) pacman_install net-tools # network tools (netstat for security auditing) pacman_install smartmontools # monitors hard drives + pacman_install lm_sensors # temperature sensors (maintenance console) + pacman_install fwupd # firmware update checks (maintenance console) pacman_install lynis # security auditing tool pacman_install telegram-desktop # messenger application # LaTeX - minimal set for document compilation with latexmk diff --git a/configs/maintenance-thresholds.toml b/configs/maintenance-thresholds.toml index 161b4c2..c14eaec 100644 --- a/configs/maintenance-thresholds.toml +++ b/configs/maintenance-thresholds.toml @@ -38,17 +38,21 @@ timeline_weekly = 2 timeline_monthly = 2 timeline_quarterly = 0 timeline_yearly = 0 +timeline_slack = 2 # grade at summed limits + slack: absorbs the + # hourly create/:45-cleanup gap; a dead cleanup + # timer still clears it within a few hours single_keep = 2 # DELETE STALE keeps the newest N singles zfs_count_warn = 1000 # runaway retention zfs_space_warn_pct = 20 # snapshot space vs pool capacity [packages] -cache_warn_gb = 10 # suggest paccache beyond the weekly keep-3 +cache_warn_gb = 20 # above keep-3 saturation (~17-19GB on ratio); + # fires within ~2 months if the paccache timer dies keyring_warn_days = 60 # stale archlinux-keyring silently breaks updates -[security] -cve_min_warn = "high" # arch-audit severity floor that warns (the - # constant Unknown/Low tail is background) +# [security] cve_min_warn retired 2026-07-08: the cve grade now keys on +# actionability (a fixed release exists and isn't installed), not severity. +# Unfixable advisories are informational and never warn. [updates] pending_warn = 50 # "a lot" — reddens the strip border @@ -77,7 +81,10 @@ expected_timers_extra = [] app_log_warn_days = 7 # log-cleanup cron should hold ~7 days journal_digest_top = 10 # groups shown in the journal digest journal_disk_warn_gb = 2 # journald footprint before vacuum suggested -coredump_window_days = 14 # dumps older than this age out of the count +coredump_window_days = 14 # forensic window (matches systemd's 2w + # tmpfiles aging): the value/grade counts + # corefiles PAST it — CLEAR's reclaim set; + # recent crashes are digest-visible only [memory] used_warn_pct = 90 # 1 - available/total (reclaimable-aware) @@ -165,4 +172,19 @@ entries = [ "/usr/lib/ghc-*/lib/package.conf.d/package.cache", # regenerated by ghc-pkg "/usr/lib/vlc/plugins/plugins.dat", # regenerated plugin cache "/etc/conf.d/pacman-contrib", # archsetup sets PACCACHE_ARGS + # Runtime perm/owner drift the system re-applies within seconds of a + # reinstall (verified on ratio 2026-07-08: reinstall reset these, the + # daemons re-drifted them before the next -Qkk): + "/etc/cups/classes.conf", # cupsd rewrites mode 600 + "/etc/cups/printers.conf", # (may carry device creds) + "/etc/cups/subscriptions.conf", + "/var/lib/passim*", # StateDirectory re-chowns + "/var/log/journal", # tmpfiles re-groups + # /etc/ssl/private deliberately NOT here: both hosts had it world-listable + # (755 vs packaged 700) — a real misconfig the metric caught. pacman warns + # but never resets existing dir perms; fixed by chmod 2026-07-08. + "/usr/lib/utempter/utempter", # setgid-helper ownership; + # NOTE: also silences a + # content change on this + # one binary (accepted) ] diff --git a/docs/homelab-inventory/2026-01-05-truenas-hardware-specs.org b/docs/homelab-inventory/2026-01-05-truenas-hardware-specs.org new file mode 100644 index 0000000..42d33ad --- /dev/null +++ b/docs/homelab-inventory/2026-01-05-truenas-hardware-specs.org @@ -0,0 +1,229 @@ +#+TITLE: TrueNAS Server Hardware Specifications +#+AUTHOR: Auto-generated via SSH +#+DATE: 2026-01-05 + +* Package Management Note + +**IMPORTANT:** Package management tools (apt) are disabled on TrueNAS appliances. Attempting to install packages with apt or methods other than the TrueNAS web interface can result in a nonfunctional system. + +All software installation and updates must be done through the TrueNAS web interface. + +* System Information + +** Hostname: truenas +** IP Address: 192.168.86.5 (static) +** Network Interface: eno1 +** Kernel: Linux 6.12.33-production+truenas +** Build Date: Wed Dec 17 21:17:21 UTC 2025 +** Architecture: x86_64 GNU/Linux + +* CPU Specifications + +** Model: Intel(R) Core(TM) i9-9900K CPU @ 3.60GHz +** Architecture: x86_64 (Coffee Lake S) +** Cores: 8 physical cores +** Threads: 16 (2 threads per core) +** Base Clock: 3.60 GHz +** Max Turbo: 5.00 GHz +** Min Clock: 800 MHz + +** Cache: +- L1d cache: 256 KiB (8 instances) +- L1i cache: 256 KiB (8 instances) +- L2 cache: 2 MiB (8 instances) +- L3 cache: 16 MiB (1 instance) + +** Features: +- Virtualization: VT-x +- CPU family: 6 +- Model: 158 +- Stepping: 13 + +* Memory Specifications + +** Total RAM: 64 GiB (62 GiB usable) +** Swap: 0 B (no swap configured) +** Current Usage: ~3.0 GiB used, 59 GiB free + +* Storage Controllers + +** SATA Controllers: +1. Intel Corporation Cannon Lake PCH SATA AHCI Controller (rev 10) +2. ASMedia Technology Inc. ASM1166 Serial ATA Controller (rev 02) + +** NVMe Controllers: +1. Sandisk Corp WD PC SN810 / Black SN850 NVMe SSD (rev 01) + +* Block Devices + +** Hard Drives: +- sda: 10.9 TB (ZFS member) +- sdb: 10.9 TB (ZFS member) +- sdc: 10.9 TB (ZFS member) +- sdd: 10.9 TB (ZFS member) +- sdg: 18.2 TB (ZFS member) +- sdh: 18.2 TB (ZFS member) +- sdi: 18.2 TB (ZFS member) +- sdj: 18.2 TB (ZFS member) + +** System Drives: +- sde: 1.9 TB (Boot pool, ZFS member) + - sde1: 1M partition + - sde2: 512M VFAT partition + - sde3: 1.9T ZFS partition +- sdf: 1.9 TB (ZFS member) + +** NVMe: +- nvme0n1: 1.8 TB + +** Total Raw Capacity: +- 4× 10.9 TB = 43.6 TB +- 4× 18.2 TB = 72.8 TB +- 2× 1.9 TB = 3.8 TB +- 1× 1.8 TB NVMe = 1.8 TB +- **Total: ~122 TB raw storage** + +* Network Interfaces + +** eno1 (Primary - Active): +- Type: Ethernet controller - Intel Corporation Ethernet Connection (7) I219-V (rev 10) +- MAC: 70:85:c2:db:9d:94 +- IP: 192.168.86.5/21 +- Subnet broadcast: 192.168.87.255 +- State: UP +- MTU: 1500 +- Alternative name: enp0s31f6 + +** enp2s0 (Secondary - Down): +- Type: Ethernet controller - Intel Corporation I211 Gigabit Network Connection (rev 03) +- MAC: 70:85:c2:db:9d:94 +- State: DOWN (NO-CARRIER) + +** Wireless: +- Intel Corporation Dual Band Wireless-AC 3168NGW [Stone Peak] (rev 10) +- Status: Not configured + +** Docker Bridge: +- docker0: 172.16.0.1/24 +- State: DOWN (NO-CARRIER) + +* Motherboard/Chipset + +** Chipset: Intel Z390 +- Host Bridge: 8th/9th Gen Core 8-core Desktop Processor (Coffee Lake S) +- ISA Bridge: Z390 Chipset LPC/eSPI Controller (rev 10) +- PCIe Root Ports: Multiple Cannon Lake PCH PCI Express ports + +** Integrated Components: +- Graphics: Intel CoffeeLake-S GT2 [UHD Graphics 630] (rev 02) +- Audio: Intel Cannon Lake PCH cAVS (rev 10) +- USB: Cannon Lake PCH USB 3.1 xHCI Host Controller (rev 10) +- Thermal: Cannon Lake PCH Thermal Controller (rev 10) +- HECI: Cannon Lake PCH HECI Controller (rev 10) +- SMBus: Cannon Lake PCH SMBus Controller (rev 10) +- SPI: Cannon Lake PCH SPI Controller (rev 10) + +* ZFS Pool Configuration + +** boot-pool (1.86 TB): +- Allocated: 3.06 GB +- Free: 1.86 TB +- Capacity: 0% +- Health: ONLINE + +** sysdata (1.86 TB): +- Allocated: 314 MB +- Free: 1.86 TB +- Capacity: 0% +- Health: ONLINE +- Purpose: System data and applications + +** tank (72.8 TB): +- Allocated: 1.17 MB +- Free: 72.7 TB +- Capacity: 0% +- Health: ONLINE +- Purpose: Large storage pool (empty, ready for expansion) + +** vault (43.6 TB): +- Allocated: 24.5 TB +- Free: 19.1 TB +- Capacity: 56% +- Health: ONLINE +- Purpose: Main media storage + +* Vault Datasets + +** Media (Movies, Music, TV Shows): +- Size: 30 TB (allocated) +- Used: 17 TB +- Available: 14 TB +- Capacity: 54% + +** Lectures: +- Size: 15 TB (allocated) +- Used: 878 GB +- Available: 14 TB +- Capacity: 6% + +** Audiobooks: +- Size: 14 TB (allocated) +- Used: 89 GB +- Available: 14 TB +- Capacity: 1% + +** Magic: +- Size: 15 TB (allocated) +- Used: 543 GB +- Available: 14 TB +- Capacity: 4% + +** Books: +- Size: 14 TB (allocated) +- Used: 177 GB +- Available: 14 TB +- Capacity: 2% + +* Security Mitigations + +** CPU Vulnerabilities (Status): +- Gather data sampling: Mitigated (Microcode) +- Indirect target selection: Mitigated (Aligned branch/return thunks) +- Itlb multihit: KVM Mitigation (Split huge pages) +- L1tf: Not affected +- Mds: Not affected +- Meltdown: Not affected +- Mmio stale data: Mitigated (Clear CPU buffers; SMT vulnerable) +- Reg file data sampling: Not affected +- Retbleed: Mitigated (Enhanced IBRS) +- Spec rstack overflow: Not affected +- Spec store bypass: Mitigated (Disabled via prctl) +- Spectre v1: Mitigated (usercopy/swapgs barriers) +- Spectre v2: Mitigated (Enhanced/Automatic IBRS) +- Srbds: Mitigated (Microcode) +- Tsx async abort: Mitigated (TSX disabled) + +* Full PCI Device List + +#+BEGIN_EXAMPLE +00:00.0 Host bridge: Intel Corporation 8th/9th Gen Core 8-core Desktop Processor Host Bridge/DRAM Registers [Coffee Lake S] (rev 0d) +00:01.0 PCI bridge: Intel Corporation 6th-10th Gen Core Processor PCIe Controller (x16) (rev 0d) +00:02.0 VGA compatible controller: Intel Corporation CoffeeLake-S GT2 [UHD Graphics 630] (rev 02) +00:12.0 Signal processing controller: Intel Corporation Cannon Lake PCH Thermal Controller (rev 10) +00:14.0 USB controller: Intel Corporation Cannon Lake PCH USB 3.1 xHCI Host Controller (rev 10) +00:14.2 RAM memory: Intel Corporation Cannon Lake PCH Shared SRAM (rev 10) +00:16.0 Communication controller: Intel Corporation Cannon Lake PCH HECI Controller (rev 10) +00:17.0 SATA controller: Intel Corporation Cannon Lake PCH SATA AHCI Controller (rev 10) +00:1c.0 PCI bridge: Intel Corporation Cannon Lake PCH PCI Express Root Port #6 (rev f0) +00:1c.6 PCI bridge: Intel Corporation Cannon Lake PCH PCI Express Root Port #7 (rev f0) +00:1d.0 PCI bridge: Intel Corporation Cannon Lake PCH PCI Express Root Port #9 (rev f0) +00:1f.0 ISA bridge: Intel Corporation Z390 Chipset LPC/eSPI Controller (rev 10) +00:1f.3 Audio device: Intel Corporation Cannon Lake PCH cAVS (rev 10) +00:1f.4 SMBus: Intel Corporation Cannon Lake PCH SMBus Controller (rev 10) +00:1f.5 Serial bus controller: Intel Corporation Cannon Lake PCH SPI Controller (rev 10) +00:1f.6 Ethernet controller: Intel Corporation Ethernet Connection (7) I219-V (rev 10) +01:00.0 SATA controller: ASMedia Technology Inc. ASM1166 Serial ATA Controller (rev 02) +02:00.0 Ethernet controller: Intel Corporation I211 Gigabit Network Connection (rev 03) +03:00.0 Network controller: Intel Corporation Dual Band Wireless-AC 3168NGW [Stone Peak] (rev 10) +04:00.0 Non-Volatile memory controller: Sandisk Corp WD PC SN810 / Black SN850 NVMe SSD (rev 01) +#+END_EXAMPLE diff --git a/docs/homelab-inventory/mybitch-laptop.org b/docs/homelab-inventory/mybitch-laptop.org new file mode 100644 index 0000000..e522c40 --- /dev/null +++ b/docs/homelab-inventory/mybitch-laptop.org @@ -0,0 +1,238 @@ +#+TITLE: mybitch - Christine's Laptop +#+DATE: 2026-01-31 +#+HOSTNAME: mybitch + +* Automated Capabilities + +# Maintained by the system-health-check workflow. Manual edits may be +# overwritten when the workflow detects drift between this drawer and the +# live system. Machine-readable capability signals used to dispatch which +# checks run on this host. + +:PROPERTIES: +:FS: ext4 +:PM: apt +:ORCH: none +:SNAPSHOT: timeshift +:MESH: tailscale +:BACKUP: source +:VIRT: libvirt +:INIT: systemd +:LAST_AUDIT: 2026-06-23 +:END: + +* Overview + +| Field | Value | +|----------+-----------------------------------| +| Hostname | mybitch | +|----------+-----------------------------------| +| Type | Laptop | +|----------+-----------------------------------| +| Owner | Christine | +|----------+-----------------------------------| +| Maker | Framework | +|----------+-----------------------------------| +| Model | Laptop 16 (AMD Ryzen 7040 Series) | +|----------+-----------------------------------| +| OS | Linux Mint 22.3 (Zena) | +|----------+-----------------------------------| +| Kernel | 6.17.0-23-generic | +|----------+-----------------------------------| +| Status | In use | +|----------+-----------------------------------| + +* Hardware Specifications + +** CPU + +| Field | Value | +|--------------+-----------------------------------| +| Model | AMD Ryzen 9 7940HS w/ Radeon 780M | +|--------------+-----------------------------------| +| Architecture | Zen 4 (TSMC 5nm) | +|--------------+-----------------------------------| +| Cores | 8 | +|--------------+-----------------------------------| +| Threads | 16 | +|--------------+-----------------------------------| +| Base Clock | 4.0 GHz | +|--------------+-----------------------------------| +| Boost Clock | 5.25 GHz | +|--------------+-----------------------------------| +| L2 Cache | 8 MiB (8x1024 KiB) | +|--------------+-----------------------------------| +| L3 Cache | 16 MiB | +|--------------+-----------------------------------| + +** Memory + +| Field | Value | +|-----------+------------------------------| +| Total | 32 GB | +|-----------+------------------------------| +| Type | DDR5-5600 | +|-----------+------------------------------| +| Installed | 1x 32GB (A-DATA AD5S560032G) | +|-----------+------------------------------| +| Slots | 2 (1 free) | +|-----------+------------------------------| +| Max | 64 GB | +|-----------+------------------------------| + +** Graphics + +| Device | Model | Driver | Notes | +|------------+---------------------+--------+-----------------| +| Dedicated | AMD Radeon RX 7700S | amdgpu | RDNA 3, Navi 33 | +|------------+---------------------+--------+-----------------| +| Integrated | AMD Radeon 780M | amdgpu | RDNA 3, Phoenix | +|------------+---------------------+--------+-----------------| + +** Storage + +| Device | Model | Size | Interface | +|--------------+---------------------+----------+--------------| +| /dev/nvme0n1 | WD BLACK SN770M 2TB | 1.82 TiB | PCIe Gen4 x4 | +|--------------+---------------------+----------+--------------| +| /dev/nvme1n1 | WD BLACK SN770 2TB | 1.82 TiB | PCIe Gen4 x4 | +|--------------+---------------------+----------+--------------| +| Total | | 3.64 TiB | | +|--------------+---------------------+----------+--------------| + +*** Partitions + +| Partition | Size | Used | Filesystem | Mount | Label | +|----------------+----------+-------------+------------+-----------+--------| +| /dev/nvme1n1p2 | 1.82 TiB | 17.4% | ext4 | / | | +|----------------+----------+-------------+------------+-----------+--------| +| /dev/nvme1n1p1 | 512 MiB | 1.2% | vfat | /boot/efi | | +|----------------+----------+-------------+------------+-----------+--------| +| /dev/nvme0n1p1 | 1.82 TiB | (unmounted) | ext4 | | Backup | +|----------------+----------+-------------+------------+-----------+--------| + +** Display + +| Field | Value | +|------------+--------------------| +| Panel | BOE Display 0x0bc9 | +|------------+--------------------| +| Resolution | 2560x1600 | +|------------+--------------------| +| Size | 16" (345x215mm) | +|------------+--------------------| +| Ratio | 16:10 | +|------------+--------------------| +| DPI | 188 | +|------------+--------------------| + +** Battery + +| Field | Value | +|----------+------------------| +| Model | NVT FRANDBA | +|----------+------------------| +| Capacity | 85.1 Wh (design) | +|----------+------------------| +| Current | 87.5 Wh (102.9%) | +|----------+------------------| +| Cycles | 13 | +|----------+------------------| +| Type | Li-ion | +|----------+------------------| + +** Network + +| Interface | Type | Chipset | +|-----------+--------+--------------------------| +| wlp5s0 | WiFi | MediaTek MT7922 802.11ax | +|-----------+--------+--------------------------| +| Bluetooth | BT 5.2 | MediaTek Wireless_Device | +|-----------+--------+--------------------------| + +** Expansion Cards (Framework) + +| Slot | Card | +|------+---------------------| +| 1 | HDMI Expansion Card | +|------+---------------------| + +** Input Modules (Framework) + +| Module | Notes | +|----------+-------------------------| +| Keyboard | ANSI layout | +|----------+-------------------------| +| Numpad | Laptop 16 Numpad Module | +|----------+-------------------------| + +* Connected Peripherals + +| Device | Connection | +|---------------------------+-------------------------| +| Logitech MX Master 3 | Unifying Receiver (USB) | +|---------------------------+-------------------------| +| Realtek Laptop Camera | Internal USB | +|---------------------------+-------------------------| +| Goodix Fingerprint Reader | Internal USB | +|---------------------------+-------------------------| + +* Network Access + +| Method | Address | +|--------+---------------| +| mDNS | mybitch.local | +|--------+---------------| +| SSH | Yes (OpenSSH) | +|--------+---------------| + +* Notes + +- Framework Laptop 16 with modular expansion card system +- Discrete AMD GPU (RX 7700S) in addition to integrated Radeon 780M +- Second NVMe slot contains 2TB backup drive (unmounted, labeled "Backup") +- Uptime at inventory: 4 days + +* Operational Changes Log + +Deliberate, non-default config changes applied to mybitch (so they can be found and reverted later). Most recent first. + +** 2026-05-12 — amdgpu.mes=0 added (escalation of the MES-hang freeze mitigation) + +The 2026-05-11 =amdgpu.cwsr_enable=0= mitigation didn't reduce the iGPU "MES failed to respond" errors (14 hits in 14.5h after the fix vs. 38 over 10 days before — rate went *up*), so escalated per the documented path: added =amdgpu.mes=0= to =GRUB_CMDLINE_LINUX_DEFAULT= in =/etc/default/grub= (now =quiet splash amdgpu.cwsr_enable=0 amdgpu.mes=0=). Backup: =/etc/default/grub.bak-2026-05-12-amdgpu-mes=. =update-grub= run; param verified in all 6 =/boot/grub/grub.cfg= entries. Takes effect on next reboot (planned for after the in-progress first rsyncshot backup). =amdgpu.mes=0= disables the GFX11 hardware MES scheduler and falls back to the legacy KIQ path — sidesteps the hanging MES firmware entirely. Tradeoff: KIQ-on-GFX11 is a less-traveled config (small chance of cosmetic display/modeset quirks); irrelevant for Christine's light desktop workload. =cwsr_enable=0= kept (harmless alongside mes=0). *Revert:* restore =/etc/default/grub.bak-2026-05-12-amdgpu-mes= (or sed out = amdgpu.mes=0=), =update-grub=, reboot. After the reboot, check =journalctl -k -b | grep -c 'MES failed'= → should be 0. See the 2026-05-11 entry in =.ai/project-workflows/system-health-check.org='s Known Issues Log for the full background. + +** 2026-05-12 — rsyncshot backups + travel keep-awake mode + +Set up the same =rsyncshot= → TrueNAS backup mybitch's siblings use, and made mybitch never suspend so backups keep running while Christine travels with it. + +*** rsyncshot install (mirrors ratio's setup) +- =/usr/local/bin/rsyncshot= — the script (source of truth: =~/code/rsyncshot/rsyncshot=). +- =/etc/rsyncshot/config= — =REMOTE_HOST="cjennings@truenas.tailf3bb8c.ts.net"=, =REMOTE_PATH="/mnt/vault/backups"=, =SSH_IDENTITY_FILE="/root/.ssh/id_ed25519"=. (See "destination pinned to Tailscale" below for why the full MagicDNS name.) +- =/etc/rsyncshot/include.txt= — =/home /etc /usr/local/bin=. =/etc/rsyncshot/exclude.txt= — copied verbatim from ratio. +- =/etc/logrotate.d/rsyncshot= — copied verbatim from ratio. =/var/log/rsyncshot.log= — the log. +- Root crontab: hourly at :30 (hours 0-1,3-23, keep 23) + daily at 2:30 (keep 30), each wrapped in =flock -x /tmp/rsyncshot.lock=. Identical to ratio's schedule. +- =/root/.ssh/id_ed25519= (+ =.pub=) — new keypair generated on mybitch, no passphrase, comment =root@mybitch-rsyncshot=. mybitch's =cjennings= account had no outbound key (it's a receive-only account), so the backup runs with a dedicated root key instead. The pubkey was appended to =~cjennings/.ssh/authorized_keys= on TrueNAS (the line tagged =root@mybitch-rsyncshot=). +- =/root/.ssh/config= — =Host truenas truenas.tailf3bb8c.ts.net 100.67.22.65= → =User cjennings=, that IdentityFile, =ServerAliveInterval 60 / ServerAliveCountMax 10 / TCPKeepAlive yes=. Host keys for those names added to =/root/.ssh/known_hosts=. +- TrueNAS side: =/mnt/vault/backups/mybitch/= directory created, owned =cjennings:cjennings= (=vault/backups= is a ZFS dataset; per-host backups are just subdirectories, like =ratio/= and =velox/=). +- *Revert:* =sudo crontab -r= (or remove the rsyncshot lines), =sudo rm -rf /usr/local/bin/rsyncshot /etc/rsyncshot /etc/logrotate.d/rsyncshot /var/log/rsyncshot.log /root/.ssh/id_ed25519* /root/.ssh/config=, restore =/root/.ssh/known_hosts= if desired, remove the =root@mybitch-rsyncshot= line from =~cjennings/.ssh/authorized_keys= on TrueNAS, and =rm -rf /mnt/vault/backups/mybitch= on TrueNAS. + +*** destination pinned to Tailscale +=REMOTE_HOST= uses the full Tailscale MagicDNS name =truenas.tailf3bb8c.ts.net= (not the bare =truenas=) so the backup always routes over the tailnet regardless of what a hotel/away-from-home router's DNS resolves =truenas= to. Tailscale uses a direct LAN path between same-LAN peers, so this is near-zero perf cost when mybitch is on the home network. *Revert:* set =REMOTE_HOST="cjennings@truenas"= in =/etc/rsyncshot/config=. + +*** keep-awake (never suspend) — three layers +1. =systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target= — the hard guarantee; suspend can't happen even if requested. *Revert:* =sudo systemctl unmask sleep.target suspend.target hibernate.target hybrid-sleep.target=. +2. Cinnamon power gsettings for user =cciarm= set to ='nothing'=: =sleep-inactive-ac-type=, =sleep-inactive-battery-type=, =lid-close-ac-action=, =lid-close-battery-action= (schema =org.cinnamon.settings-daemon.plugins.power=). Were ='suspend'/'suspend'/'nothing'/'suspend'=. *Revert:* set them back, or run =gsettings reset= on each key as =cciarm=. +3. =/etc/systemd/logind.conf.d/no-suspend-keep-awake.conf= — =HandleLidSwitch=ignore=, =HandleLidSwitchExternalPower=ignore=, =HandleLidSwitchDocked=ignore=, =IdleAction=ignore=. (Written but not applied live — =systemd-logind= wasn't restarted because sessions were active; takes effect on next reboot. Layers 1-2 cover the gap.) *Revert:* =sudo rm /etc/systemd/logind.conf.d/no-suspend-keep-awake.conf= then reboot or =systemctl restart systemd-logind=. + +When mybitch goes back to being a stationary machine (or this whole arrangement is no longer wanted), reverse layers 1-3. The rsyncshot backup itself is worth keeping regardless. + +*** IPv6 set to link-local-only (fixes the tailscale dropout) +mybitch dropped off the Tailscale tailnet on 2026-05-12 AM. Root cause: the home WiFi ("dupre") has device(s) advertising ULA-only IPv6 RAs (prefixes =fdc4:c4c6:3fb0:16cc::/64= and =fd67:8615:6780:1::/64=) with no default route. NetworkManager (=ipv6.method=auto=) SLAAC'd a "global"-scope ULA address onto wlp5s0 with no v6 internet path, so tailscaled kept trying IPv6 to controlplane/DERP, failed "network is unreachable", flapped, and lost its control connection. mybitch has no working global IPv6 anywhere (no ISP v6), so the fix is to ignore underlay v6: +- =/etc/NetworkManager/conf.d/ipv6-link-local-only.conf= → =[connection] ipv6.method=link-local= (connection default — covers home WiFi, travel WiFi, phone hotspot). +- =nmcli con modify dupre ipv6.method link-local= (explicit on the home connection). +- Effect: wlp5s0 keeps only fe80:: (mDNS still works), no SLAAC, no v6 routes; tailscale0's overlay v6 untouched; tailscaled uses the IPv4 underlay. Activates on reboot or =nmcli dev reapply wlp5s0=. +- *Revert if mybitch ever gets real IPv6:* =sudo rm /etc/NetworkManager/conf.d/ipv6-link-local-only.conf=; =sudo nmcli con modify dupre ipv6.method auto=; reboot or =nmcli dev reapply wlp5s0=. + +*** caveats (relayed to Craig for Christine) +- mybitch must stay plugged into AC for backups to keep running — masking suspend doesn't stop a battery dying with the lid closed. +- mybitch must be on WiFi at the destination and stay on the Tailscale tailnet. diff --git a/docs/homelab-inventory/ratio-desktop.org b/docs/homelab-inventory/ratio-desktop.org new file mode 100644 index 0000000..234c1a4 --- /dev/null +++ b/docs/homelab-inventory/ratio-desktop.org @@ -0,0 +1,175 @@ +#+TITLE: ratio - Desktop Workstation +#+DATE: 2026-01-27 +#+HOSTNAME: ratio + +* Automated Capabilities + +# Maintained by the system-health-check workflow. Manual edits may be +# overwritten when the workflow detects drift between this drawer and the +# live system. Machine-readable capability signals used to dispatch which +# checks run on this host. + +:PROPERTIES: +:FS: btrfs +:PM: pacman +:ORCH: topgrade +:SNAPSHOT: snapper +:MESH: tailscale +:BACKUP: source +:VIRT: libvirt,docker,podman +:INIT: systemd +:LAST_AUDIT: 2026-07-08 +:END: + +* Overview + +| Field | Value | +|-------------+------------------------------------------| +| Hostname | ratio | +|-------------+------------------------------------------| +| Type | Desktop workstation | +|-------------+------------------------------------------| +| Maker | Framework | +|-------------+------------------------------------------| +| Model | Desktop (AMD Ryzen AI Max 300 Series) | +|-------------+------------------------------------------| +| Board | FRANMFCP06 v A6 | +|-------------+------------------------------------------| +| Firmware | INSYDE UEFI v03.03 (2025-09-16) | +|-------------+------------------------------------------| +| OS | Arch Linux | +|-------------+------------------------------------------| +| Kernel | 6.12.67-1-lts | +|-------------+------------------------------------------| +| Desktop | Hyprland (Wayland) | +|-------------+------------------------------------------| +| Previous ID | cogito (repurposed and renamed to ratio) | +|-------------+------------------------------------------| + +* CPU + +| Field | Value | +|---------------+---------------------------------------| +| Model | AMD Ryzen AI MAX+ 395 w/ Radeon 8060S | +|---------------+---------------------------------------| +| Architecture | Zen 5 (TSMC 4nm) | +|---------------+---------------------------------------| +| Cores/Threads | 16 cores / 32 threads | +|---------------+---------------------------------------| +| Base/Boost | 3.0 GHz / 5.15 GHz | +|---------------+---------------------------------------| +| Cache L1 | 1.2 MiB (d-16x48 KiB; i-16x32 KiB) | +|---------------+---------------------------------------| +| Cache L2 | 16 MiB (16x1024 KiB) | +|---------------+---------------------------------------| +| Cache L3 | 64 MiB (2x32 MiB) | +|---------------+---------------------------------------| +| NPU | XDNA 2, 50+ peak AI TOPS | +|---------------+---------------------------------------| +| Peak Perf | 59.4 FP16/BF16 TFLOPS @ 2.9GHz | +|---------------+---------------------------------------| + +* GPU + +| Field | Value | +|--------------+-----------------------------------------| +| Model | AMD Radeon 8060S (Strix Halo) | +|--------------+-----------------------------------------| +| Architecture | RDNA 3.5 | +|--------------+-----------------------------------------| +| CUs | 40 Compute Units | +|--------------+-----------------------------------------| +| Driver | amdgpu (kernel) | +|--------------+-----------------------------------------| +| Max VRAM | 96GB (via AMD Variable Graphics Memory) | +|--------------+-----------------------------------------| +| GPU Arch ID | gfx1151 | +|--------------+-----------------------------------------| +| PCIe | Gen 4, 16 GT/s, 16 lanes | +|--------------+-----------------------------------------| + +* Memory + +| Field | Value | +|--------------+--------------------------| +| Total | 128 GiB unified (LPDDR5) | +|--------------+--------------------------| +| Speed | 8000 MT/s | +|--------------+--------------------------| +| Channels | 8 (8x 16 GiB) | +|--------------+--------------------------| +| Manufacturer | Micron Technology | +|--------------+--------------------------| +| Part Number | MT62F4G32D8DV-023 WT | +|--------------+--------------------------| + +* Storage + +| Device | Model | Size | Interface | +|--------------+------------------------+-----------+-----------| +| /dev/nvme0n1 | WD BLACK SN850X 8000GB | 7.28 TiB | NVMe | +|--------------+------------------------+-----------+-----------| +| /dev/nvme1n1 | WD BLACK SN850X 8000GB | 7.28 TiB | NVMe | +|--------------+------------------------+-----------+-----------| +| *Total* | | 14.55 TiB | | +|--------------+------------------------+-----------+-----------| + +Filesystem: btrfs with subvolumes (@, @home, @snapshots, @log, @pkg) + +* Network + +| Interface | Chipset | Speed | Type | +|-----------+-----------------+---------+----------| +| enp191s0 | Realtek RTL8126 | 5 GbE | Wired | +|-----------+-----------------+---------+----------| +| wlp192s0 | MediaTek MT7925 | Wi-Fi 7 | Wireless | +|-----------+-----------------+---------+----------| + +Also has: docker0, tailscale0, virbr0 (virtual interfaces) + +* Audio + +- AMD Radeon High Definition Audio (HDMI, snd_hda_intel) +- AMD Ryzen HD Audio (3.5mm, snd_hda_intel) +- PipeWire v1.4.10 (pipewire-pulse, wireplumber) + +* PCI Slots + +| Slot | Info | Status | +|------+------------+-----------| +| 1 | M.2 JWLAN | Available | +|------+------------+-----------| +| 2 | M.2 JSSD1 | In use | +|------+------------+-----------| +| 3 | M.2 JSSD2 | In use | +|------+------------+-----------| +| 4 | PCIe Gen 4 | Available | +|------+------------+-----------| + +* Connected Peripherals + +- Monitor: Dell U3419W (HDMI-A-1, 3440x1440@60Hz) +- Keyboard: Das Keyboard 4 (Cherry MX Blue) — replacing with Keychron Q6 Pro +- Mouse: Logitech M650 +- DAC/Amp: JDS Labs Element IV (ordered) + +* AI/LLM Capability + +Primary purpose includes local AI/LLM inference via Ollama. + +| Model | Size | Purpose | +|------------------+-------+----------------------| +| deepseek-r1:70b | 42 GB | Main reasoning model | +|------------------+-------+----------------------| +| qwen3:30b | 18 GB | MoE chat (256K ctx) | +|------------------+-------+----------------------| +| deepseek-r1:14b | 9 GB | Light reasoning | +|------------------+-------+----------------------| +| nomic-embed-text | 274MB | RAG embeddings | +|------------------+-------+----------------------| + +Inference benchmarks (AMD testing, LM Studio 0.3.11): +- Small models (1-3B): ~100+ tok/s +- Medium models (7-8B): ~60-80 tok/s +- Large models (20B): ~58 tok/s +- Very large models (120B): ~38 tok/s diff --git a/docs/homelab-inventory/truenas-server.org b/docs/homelab-inventory/truenas-server.org new file mode 100644 index 0000000..5ce8613 --- /dev/null +++ b/docs/homelab-inventory/truenas-server.org @@ -0,0 +1,114 @@ +#+TITLE: truenas - NAS/Storage Server +#+DATE: 2026-01-27 +#+HOSTNAME: truenas + +* Automated Capabilities + +# Maintained by the system-health-check workflow. Manual edits may be +# overwritten when the workflow detects drift between this drawer and the +# live system. Machine-readable capability signals used to dispatch which +# checks run on this host. +# +# NOTE: TrueNAS SCALE is middleware-managed. No user-level package manager +# or topgrade orchestration. Snapshot and update operations go through the +# TrueNAS UI / API, not pacman/apt. + +:PROPERTIES: +:FS: zfs +:PM: none +:ORCH: none +:SNAPSHOT: zfs-native +:MESH: tailscale +:BACKUP: target +:VIRT: docker +:INIT: systemd +:LAST_AUDIT: 2026-04-21 +:END: + +* Overview + +| Field | Value | +|----------+-----------------------------------------| +| Hostname | truenas (truenas.local / 192.168.86.5) | +|----------+-----------------------------------------| +| Type | NAS / storage server | +|----------+-----------------------------------------| +| OS | TrueNAS | +|----------+-----------------------------------------| +| Purpose | Media storage, backups, Plex, Syncthing | +|----------+-----------------------------------------| + +See [[file:2026-01-05-truenas-hardware-specs.org][TrueNAS Hardware Specs]] for full sysinfo output. + +* CPU + +| Field | Value | +|---------------+----------------------| +| Model | Intel Core i9-9900K | +|---------------+----------------------| +| Architecture | Coffee Lake | +|---------------+----------------------| +| Cores/Threads | 8 cores / 16 threads | +|---------------+----------------------| +| Base/Boost | 3.60 GHz / 5.00 GHz | +|---------------+----------------------| + +* Memory + +| Field | Value | +|-------+-------------| +| Total | 64 GiB DDR4 | +|-------+-------------| + +* Storage + +| Pool | Config | Raw Size | Purpose | +|-----------+-----------------+------------+---------------| +| boot-pool | Mirror | ~1.9 TB x2 | System | +|-----------+-----------------+------------+---------------| +| sysdata | NVMe | ~1.8 TB | System data | +|-----------+-----------------+------------+---------------| +| vault | RAIDZ1 (4 disk) | ~43.6 TB | Primary media | +|-----------+-----------------+------------+---------------| +| tank | RAIDZ1 (4 disk) | ~72.8 TB | Expansion | +|-----------+-----------------+------------+---------------| + +Total raw: ~122 TB + +*** Vault Datasets +| Dataset | Size | +|------------+-------| +| Media | 30 TB | +|------------+-------| +| Lectures | 15 TB | +|------------+-------| +| Audiobooks | 14 TB | +|------------+-------| +| Magic | 15 TB | +|------------+-------| +| Books | 14 TB | +|------------+-------| + +* Network + +| Interface | Chipset | Speed | Status | +|-----------+--------------+--------+--------------| +| eno1 | Intel I219-V | 1 GbE | Current | +|-----------+--------------+--------+--------------| +| enp2s0 | Intel I211 | 1 GbE | Available | +|-----------+--------------+--------+--------------| +| (pending) | Intel I226-V | 2.5GbE | Ordered, NIC | +|-----------+--------------+--------+--------------| + +* Motherboard + +| Field | Value | +|---------+------------| +| Chipset | Intel Z390 | +|---------+------------| + +* Services + +- Plex Media Server +- Syncthing +- Tailscale diff --git a/docs/homelab-inventory/velox-laptop.org b/docs/homelab-inventory/velox-laptop.org new file mode 100644 index 0000000..6e4f540 --- /dev/null +++ b/docs/homelab-inventory/velox-laptop.org @@ -0,0 +1,158 @@ +#+TITLE: velox - Laptop +#+DATE: 2026-01-27 +#+HOSTNAME: velox + +* Automated Capabilities + +# Maintained by the system-health-check workflow. Manual edits may be +# overwritten when the workflow detects drift between this drawer and the +# live system. Machine-readable capability signals used to dispatch which +# checks run on this host. +# +# NOTE: 2026-04 velox was reinstalled Arch-on-ZFS. Values below reflect +# that reinstall but remain TBD where a live probe has not yet confirmed. + +:PROPERTIES: +:FS: zfs +:PM: pacman +:ORCH: topgrade +:SNAPSHOT: sanoid +:MESH: tailscale +:BACKUP: source +:VIRT: libvirt,docker,podman +:INIT: systemd +:LAST_AUDIT: 2026-05-26 +:END: + +* Overview + +| Field | Value | +|----------+---------------------------------| +| Hostname | velox | +|----------+---------------------------------| +| Type | Laptop | +|----------+---------------------------------| +| Maker | Framework | +|----------+---------------------------------| +| Model | Laptop (13th Gen Intel Core) | +|----------+---------------------------------| +| Board | FRANMCCP07 v A7 | +|----------+---------------------------------| +| Firmware | INSYDE UEFI v03.07 (2024-12-26) | +|----------+---------------------------------| +| OS | Arch Linux | +|----------+---------------------------------| +| Kernel | 6.18.6-arch1-1 | +|----------+---------------------------------| +| Desktop | startx (X11) | +|----------+---------------------------------| + +* CPU + +| Field | Value | +|---------------+-----------------------------------------| +| Model | 13th Gen Intel Core i7-1370P | +|---------------+-----------------------------------------| +| Architecture | Raptor Lake (Intel 7 / 10nm) | +|---------------+-----------------------------------------| +| Cores/Threads | 14 cores / 20 threads (6P + 8E, hybrid) | +|---------------+-----------------------------------------| +| Base/Boost | 990 MHz / 5.2 GHz | +|---------------+-----------------------------------------| +| Cache L1 | 1.2 MiB (d-8x32K,6x48K; i-6x32K,8x64K) | +|---------------+-----------------------------------------| +| Cache L2 | 11.5 MiB (6x1.2M, 2x2M) | +|---------------+-----------------------------------------| +| Cache L3 | 24 MiB (1x24M) | +|---------------+-----------------------------------------| +| Socket | BGA1744 (U3E1) | +|---------------+-----------------------------------------| + +* GPU + +| Field | Value | +|--------------+------------------------| +| Model | Intel Iris Xe Graphics | +|--------------+------------------------| +| Architecture | Xe (Intel 7) | +|--------------+------------------------| +| Driver | i915 / modesetting | +|--------------+------------------------| + +* Memory + +| Field | Value | +|----------+------------------------------------| +| Total | 64 GiB (DDR4) | +|----------+------------------------------------| +| Speed | 3200 MT/s | +|----------+------------------------------------| +| Slots | 2 (both populated) | +|----------+------------------------------------| +| Device 1 | 32 GiB Crucial CT32G4SFD832A.C16FB | +|----------+------------------------------------| +| Device 2 | 32 GiB Crucial CT32G4SFD832A.C16FE | +|----------+------------------------------------| + +* Storage + +| Device | Model | Size | Interface | +|--------------+----------------------+----------+-----------| +| /dev/nvme0n1 | Sabrent SB-RKT4P-8TB | 7.28 TiB | NVMe | +|--------------+----------------------+----------+-----------| + +- Filesystem: ZFS on root (zroot pool on nvme0n1p2; reinstalled 2026-04) +- Datasets: zroot/ROOT/default, zroot/home, zroot/var/log, zroot/var/lib/pacman, zroot/var/lib/docker (per-layer), zroot/vms, zroot/media +- Snapshots: sanoid hourly + zfs-scrub-weekly@zroot +- SMART: PASSED (re-verified 2026-04-26); composite 43 °C; 0 media errors +- Power-on history pre-dates reinstall — refresh on next full re-inventory + +* Display + +| Field | Value | +|------------+---------------------------------| +| Panel | BOE Display 0x0bca (built 2022) | +|------------+---------------------------------| +| Resolution | 2256x1504 | +|------------+---------------------------------| +| Size | 13.5" (285x190mm) | +|------------+---------------------------------| +| DPI | 201 | +|------------+---------------------------------| +| Ratio | 3:2 | +|------------+---------------------------------| + +* Network + +| Interface | Chipset | Speed | Type | +|-------------+--------------------------------+---------+----------| +| wlp170s0 | Qualcomm Atheros AR9462 | Wi-Fi | Wireless | +|-------------+--------------------------------+---------+----------| +| enp0s13f0u3 | Realtek USB 10/100/1G/2.5G LAN | 2.5 GbE | USB | +|-------------+--------------------------------+---------+----------| + +Also has: docker0, tailscale0, virbr0 (virtual interfaces) + +* Audio + +- Intel Raptor Lake-P/U/H cAVS (snd_hda_intel) +- PipeWire v1.4.10 (pipewire-pulse, wireplumber) + +* Battery + +| Field | Value | +|-----------+-----------------------| +| Capacity | 55 Wh (design) | +|-----------+-----------------------| +| Condition | 44/55 Wh (80% health) | +|-----------+-----------------------| +| Cycles | 146 | +|-----------+-----------------------| +| Type | Li-ion | +|-----------+-----------------------| + +* Other Hardware + +- Webcam: Realtek Laptop Camera (UVC) +- Fingerprint: Shenzhen Goodix USB Device +- Bluetooth: Foxconn (BT 4.0) diff --git a/docs/specs/2026-07-07-maintenance-console-spec.org b/docs/specs/2026-07-07-maintenance-console-spec.org index 008faeb..bff187b 100644 --- a/docs/specs/2026-07-07-maintenance-console-spec.org +++ b/docs/specs/2026-07-07-maintenance-console-spec.org @@ -4,10 +4,11 @@ #+TODO: TODO | DONE #+TODO: DRAFT READY DOING | IMPLEMENTED SUPERSEDED CANCELLED -* DOING Maintenance console +* IMPLEMENTED Maintenance console :PROPERTIES: :ID: 9d9df833-c592-4aec-a7df-50d588e943ce :END: +- 2026-07-08 Wed @ 06:18:14 -0500 — IMPLEMENTED — all 14 build phases shipped (1-11 dotfiles 43a39ac..10033be, 11b 3ee22a8, 12 archsetup d6993d3 + dotfiles 0636554, 13 archsetup bef7053 + 18c081f + dotfiles 9a3f0c7). Closing state: maint CLI + GTK panel + waybar glyph live on both hosts; installer wires TOML, timers, and deps; system-health-check workflow adopted and TOML-rewired; VM scenario harness 9/9, nspawn 4/4, AT-SPI smoke green; per-phase review subagents all landed Approve. Residuals tracked in todo.org: manual-testing checklist, zfs base-image DKMS bug [#C], vNext [#D]. - 2026-07-07 Tue @ 19:54:04 -0500 — DOING — decomposed into the build: 13 phase tasks + manual-testing + flip task under the todo.org parent "Maintenance console build" (:SPEC_ID: stamped); vNext logged [#D]. - 2026-07-07 Tue @ 19:05:00 -0500 — READY — round-2 spec-review passed (all 10 round-1 findings resolved, no new blockers); Craig approved the config-paths proposal and the sysmon right-click re-homing, closing the last open decision (cookies 13/13, 10/10). - 2026-07-07 Tue @ 18:30:04 -0500 — DRAFT — drafted from the completed design arc (docs/design/maintenance-console-design-ideas.org, all decisions dated 2026-07-06/07) and the converged E5 interactive prototype. @@ -16,7 +17,7 @@ | Field | Value | |----------+-----------------------------------------------------------------------------------------------------------| -| Status | doing | +| Status | implemented | |----------+-----------------------------------------------------------------------------------------------------------| | Owner | Craig Jennings | |----------+-----------------------------------------------------------------------------------------------------------| @@ -267,6 +268,9 @@ The wall widget (lamp stream, date+time, inline results, running-%, HIDE/COPY, 3 ** Phase 11 — Waybar glyph + timers =indicator.py= + =waybar-maint=, =custom/maint= replacing =custom/sysmon= (canonical config + =waybar-active-config= runtime path + SIGUSR2), signal-driven refresh, =maint-scan.timer= + =maint-net-scan.timer= user units, glyph color tracks worst diagnostic from the state file; on battery hosts the module text is the live battery level (sysfs read per waybar interval, charging indicator, low-charge threshold feeding the diagnostic state) and any existing battery module retires. Glyph CSS lands in both theme waybar stylesheets (dupre, hudson). Retirement collateral in the same commit (Craig ruled 2026-07-07): =custom/maint= right-click re-homes the btop scratchpad (=pypr toggle monitor=); the =waybar-sysmon= + =sysmon-cycle= scripts and their suites (=tests/waybar-sysmon/=, =tests/sysmon-cycle/=) retire, and the =#custom-sysmon= CSS blocks come out of =waybar/style.css= and both themes. +** Phase 11b — Prototype fidelity pass (added 2026-07-08) +Added mid-build from Craig's live-board review: phases 7-10 delivered E5's structure and behavior but rendered subpanel metrics as one-line list rows (the sibling panels' idiom) instead of the prototype's instrument-card grid; the per-phase screenshot checks verified structure, not presentation. Scope: card-grid subpanels (4-up metric cards — big-number value, caption line, progress bars, radial gauges for scrub cadence and NVMe wear, status chips, corner lever key), two-row selector tiles with compact count chips, subpanel attention/ok/fixable/watch header line, evidence digests as full-width rosters under the grid. Engine and viewmodel contracts unchanged; =gui.py= layout + =panel.css=. Sequenced before Phase 12 so the AT-SPI smoke targets the final layout. Verification is a pixel-level comparison against the settled E5 render (headless Chrome), not structural spot checks. + ** Phase 12 — VM remedy scenarios + AT-SPI smoke (archsetup + dotfiles) Scenario orchestration over the existing snapshot primitives (=vm-utils.sh:303-357=): grouped batches per Decision 11, maint scenario scripts (stop cronie, mask fstrim, orphan packages, fill cache, break timers → =maint fix= → assert post-state); systemd-nspawn variant for pacman-level cases; AT-SPI smoke on GOOD/BAD fixtures asserting board render, arm behavior, and no leaked processes, plus the enumerated =test-panel-maint= Makefile target. May split VM work (archsetup) from AT-SPI smoke (dotfiles) into two sessions — each half gates independently. diff --git a/docs/workflows/system-health-check.org b/docs/workflows/system-health-check.org new file mode 100644 index 0000000..b66a186 --- /dev/null +++ b/docs/workflows/system-health-check.org @@ -0,0 +1,1034 @@ +#+TITLE: System Health Check Workflow +#+AUTHOR: Craig Jennings & Claude +#+DATE: 2026-02-27 + +* Overview + +This workflow performs a comprehensive diagnostic scan of whatever host it runs on (ratio, velox, mybitch, or truenas via SSH). Unlike status-check.org (which monitors a single long-running job in real-time), this scans the entire system, produces a severity-ranked report, then investigates and proposes fixes one by one. + +Run this when Craig asks "how is <host> doing?", "check my system health", or as a periodic maintenance check. + +Owned by archsetup (moved from the home project 2026-07-08 — system maintenance is archsetup's domain). Canonical location: =docs/workflows/system-health-check.org= in the archsetup repo; the per-host inventories it cross-references live beside it in =docs/homelab-inventory/=. + +Relationship to the =maint= console: on the Arch daily drivers (ratio, velox), =maint status --json= collects most of Phases 0–1 in about a second from the same severity thresholds, and =maint doctor= runs the safe remedies — prefer it for routine checks there. This workflow remains the tool for the non-Arch hosts (mybitch, truenas), for forensic deep dives (Phase 2 investigation, the case files), and for the update/reboot choreography in Phase 3. + +* Severity thresholds — the TOML is authoritative + +The graded severity thresholds in this workflow are defined in the installed thresholds file, shared with the =maint= console so the two can never drift: + +- Installed (read this): =~/.config/archsetup/maintenance-thresholds.toml= +- Canonical seed: =configs/maintenance-thresholds.toml= in the archsetup repo + +Read the TOML at the start of Phase 1 and grade against its values. Where a check below names a threshold, it cites the TOML key (e.g. =storage.df_warn_pct=); any inline number is the seeded default kept for readability, and the TOML wins on disagreement. User overrides live in =~/.config/maint/curation.toml= and merge over the shipped layer. + +* Hosts & Capabilities + +The workflow is capability-dispatched: it probes the live system in Phase 0 and runs only the checks that apply (Btrfs vs ZFS vs ext4 on storage; pacman vs apt vs none on package maintenance; etc.). The per-host inventory files under =docs/homelab-inventory/= declare the expected capabilities in a property drawer and act as a cross-check for drift. + +Host inventory files (match by =#+HOSTNAME:= keyword inside each): + +| Host | Inventory file | Notes | +|---------+-------------------------------------------+----------------------------------| +| ratio | [[file:../homelab-inventory/ratio-desktop.org][docs/homelab-inventory/ratio-desktop.org]] | Arch, Btrfs RAID1, pacman | +|---------+-------------------------------------------+----------------------------------| +| velox | [[file:../homelab-inventory/velox-laptop.org][docs/homelab-inventory/velox-laptop.org]] | Arch, ZFS (re-installed 2026-04) | +|---------+-------------------------------------------+----------------------------------| +| mybitch | [[file:../homelab-inventory/mybitch-laptop.org][docs/homelab-inventory/mybitch-laptop.org]] | Mint, ext4, apt | +|---------+-------------------------------------------+----------------------------------| +| truenas | [[file:../homelab-inventory/truenas-server.org][docs/homelab-inventory/truenas-server.org]] | TrueNAS SCALE, ZFS, no PM | +|---------+-------------------------------------------+----------------------------------| + +Each inventory file contains an =* Automated Capabilities= section with a property drawer (=:FS:=, =:PM:=, =:ORCH:=, =:SNAPSHOT:=, =:MESH:=, =:BACKUP:=, =:VIRT:=, =:INIT:=, =:LAST_AUDIT:=). Phase 0 parses this drawer, compares it to the live probe, and announces + writes back any drift. + +* The Workflow + +** Phase 0: Capability Detection + +Before running any health checks, probe the live system to determine which checks apply, then cross-reference against the inventory file for this host. + +*** Probe the live system + +#+begin_src bash +# Hostname — prefer hostnamectl (always present on systemd hosts), fall back to uname +host=$(hostnamectl --static 2>/dev/null || uname -n) + +# Filesystem of / +root_fs=$(findmnt -n -o FSTYPE /) + +# Any btrfs present? +has_btrfs=$(findmnt -t btrfs -n 2>/dev/null | wc -l) + +# Any zfs pool imported? +has_zfs=$(command -v zpool >/dev/null 2>&1 && zpool list -H -o name 2>/dev/null | wc -l || echo 0) + +# Package manager +pm=none +command -v pacman >/dev/null && pm=pacman +command -v apt >/dev/null && pm=apt +command -v dnf >/dev/null && pm=dnf + +# Orchestrator +orch=none +command -v topgrade >/dev/null && orch=topgrade + +# Snapshot tool +snapshot=none +command -v snapper >/dev/null && snapshot=snapper +# ZFS-native snapshots present? +[ "$has_zfs" -gt 0 ] && zfs list -t snapshot -H 2>/dev/null | head -1 | grep -q . && snapshot=zfs-native +command -v sanoid >/dev/null && snapshot=sanoid + +# Mesh +mesh=none +command -v tailscale >/dev/null && mesh=tailscale + +# Backup role +backup=none +[ -f /var/log/rsyncshot.log ] && backup=source +# Target role is declared by inventory, not probed (target hosts are discovered by their role, not a local artifact). + +# Virt / containers +virt="" +command -v virsh >/dev/null && virt="${virt}libvirt," +command -v docker >/dev/null && virt="${virt}docker," +command -v podman >/dev/null && virt="${virt}podman," +virt="${virt%,}" +[ -z "$virt" ] && virt=none + +# Init +init=$(ps -p 1 -o comm=) + +echo "host=$host fs=$root_fs btrfs=$has_btrfs zfs=$has_zfs pm=$pm orch=$orch snapshot=$snapshot mesh=$mesh backup=$backup virt=$virt init=$init" +#+end_src + +Present the probe result as a one-line capability summary at the top of the report. + +*** Cross-reference against inventory + +Locate the inventory file for this host: + +#+begin_src bash +inv=$(grep -l "^#+HOSTNAME: $host\b" "${ARCHSETUP_DIR:-$HOME/code/archsetup}"/docs/homelab-inventory/*.org 2>/dev/null | head -1) +#+end_src + +If no inventory file matches → print =NO INVENTORY FILE for $host= as a WARNING and skip the drift check. + +If found: parse the =:PROPERTIES:= drawer under the =* Automated Capabilities= heading and compare each key to the live probe. + +*Canonical parser* (the naive awk range =/^\* Automated Capabilities/,/^\* /= does NOT work — the end pattern matches the start line, truncating the range to one line): + +#+begin_src bash +get_inv() { + local key="$1" inv="$2" + awk -v key="$key" ' + /^\* Automated Capabilities/ { in_section=1; next } + /^\* / && in_section { in_section=0 } + in_section && $0 ~ "^:"key":" { + sub("^:"key":[[:space:]]*", "") + sub("[[:space:]]+$", "") + print; exit + } + ' "$inv" +} +#+end_src + +Key-to-probed-variable mapping: + +| Key | Probed variable | +|--------------+-----------------| +| =:FS:= | =root_fs= | +|--------------+-----------------| +| =:PM:= | =pm= | +|--------------+-----------------| +| =:ORCH:= | =orch= | +|--------------+-----------------| +| =:SNAPSHOT:= | =snapshot= | +|--------------+-----------------| +| =:MESH:= | =mesh= | +|--------------+-----------------| +| =:BACKUP:= | =backup= | +|--------------+-----------------| +| =:VIRT:= | =virt= | +|--------------+-----------------| +| =:INIT:= | =init= | +|--------------+-----------------| + +*** Announce-then-update on drift + +For each mismatch (excluding =TBD= on the inventory side, which means "not yet probed"): + +1. Print in the Phase 0 summary: =DRIFT: <host> <key>: inventory=X probed=Y — updating= +2. Edit the inventory file: replace the drawer value with the probed value. +3. Update =:LAST_AUDIT:= to today's date. + +Skip the write-back if the workflow is running on a host where the repo isn't checked out (e.g., truenas accessed via SSH). In that case, print the drift but flag it as =DRIFT (read-only host): update <file> manually=. + +*** Capability summary dictates which checks run + +Each check in the Reference section declares =Applies when: <capability>=. Phase 1 runs the check only if the capability is present; otherwise it prints =N/A — <capability> not present= for that check and moves on. + +** Phase 1: Scan & Report + +Run all checks from the Health Checks Reference below. Collect every finding into a ranked table sorted by severity: + +| Severity | Meaning | +|----------+-------------------------------------------------------------| +| CRITICAL | Service down, disk failing, backups not running | +|----------+-------------------------------------------------------------| +| WARNING | Disk space low, stale logs, failed units, overdue maint | +|----------+-------------------------------------------------------------| +| INFO | Informational (uptime, versions, counts) — no action needed | +|----------+-------------------------------------------------------------| + +*Before presenting findings:* cross-reference each finding against the Known Issues Log at the bottom of this file. If a finding matches a known issue with a decided resolution (won't-fix, accepted-noise, fixed-in-config), annotate it as =KNOWN — <short note>= rather than presenting it as a fresh WARNING. This prevents re-investigating previously-decided items and keeps the report focused on actionable new state. + +Present the report as described in Report Format, then proceed to Phase 2. + +** Phase 2: Investigate & Fix + +After presenting the ranked report: + +1. Start with the highest-severity issue +2. Investigate root cause with detailed commands +3. Propose a fix, explain what it does +4. Wait for Craig's approval before applying any changes +5. Verify the fix worked +6. Move to next issue +7. Repeat until all CRITICAL and WARNING items are addressed +8. INFO items are presented for awareness only (no action unless Craig wants to dig in) + +Handle issues one at a time — don't batch fixes together. Craig prefers to approve each individually. + +*** Defer Transient Warnings Until After Phase 3 Reboot + +If the Phase 1 pending-updates list includes any of these packages, =DEFER= Phase 2 deep-dive on network/DNS/VPN/uptime-accumulation warnings — investigate only if they survive the reboot: + +| Package (any of) | Defers these warning categories | +|---------------------------+---------------------------------------------------------------| +| kernel (linux, linux-lts) | bgscan/wifi driver spam, firmware-related dmesg noise | +|---------------------------+---------------------------------------------------------------| +| iproute2 | VPN daemon failures, ip-rule/routing-related service failures | +|---------------------------+---------------------------------------------------------------| +| systemd | resolved/networkd/logind service anomalies | +|---------------------------+---------------------------------------------------------------| +| NetworkManager | DNS reachability health checks, Tailscale DNS warnings | +|---------------------------+---------------------------------------------------------------| +| wpa_supplicant | WiFi connection/scan errors | +|---------------------------+---------------------------------------------------------------| + +*Rationale:* Warnings on a long-uptime system (>7 days) frequently reflect accumulated mid-transition state from earlier package updates that weren't rebooted into. A fresh reboot resolves these automatically. Investigation pre-reboot is wasted effort. + +*Procedure:* Mark the deferred warnings in the Phase 1 report as =DEFER-TO-POST-REBOOT=. Proceed directly from Phase 2 (investigating any non-deferrable warnings) to Phase 3. After reboot (see two-stage pattern in Phase 3), re-evaluate the deferred warnings — most will have vanished. Any that survive get the full Phase 2 treatment. + +** Phase 3: System Update + +Updates are separate from issue investigation. After all issues are addressed (or deferred per Phase 2's transient-warning rule): + +1. Review the pending update list (already gathered in Phase 1) +2. Identify notable packages (major version bumps, GPU drivers, kernel, firmware) +3. Check the [[https://archlinux.org/news/][Arch Linux News]] page for any manual intervention notices +4. Check =/var/log/pacman.log= for recent failed transactions +5. *Host-specific kernel watches.* On ratio: if =linux=, =linux-lts=, =linux-firmware=, or a major =mesa= bump is pending, run the addendum at =strix-soak-watch.org= before topgrade. Retire the addendum (delete the file + this bullet) when the strix-lts custom kernel is retired. +6. Run =topgrade= for the actual update (config at =~/.config/topgrade.toml=). On maint hosts (ratio, velox) plain =topgrade= resolves to the dotfiles PATH wrapper, which stamps the console's topgrade-freshness metric on success — no extra step. If the run happened outside the wrapper somehow, =maint stamp topgrade= records it by hand. +7. If linux-firmware, kernel, or Mesa were updated, recommend a reboot + +*** Two-Stage Reboot Pattern (MANDATORY if Phase 3 installed kernel / iproute2 / systemd / NetworkManager) + +When a core networking or kernel package is updated, the workflow formally splits into two sessions: + +*Stage 1 — Pause before reboot:* +1. Update =docs/session-context.org= with: + - What's been completed (Phase 1 findings, Phase 3 summary, packages bumped) + - What's deferred to post-reboot (the =DEFER-TO-POST-REBOOT= warnings from Phase 2) + - Remaining work: Phase 4, wrap-up +2. Tell Craig to reboot +3. End session (do NOT delete session-context.org — its presence signals an in-progress health check) + +*Stage 2 — Resume in new session post-reboot:* +1. Read =docs/session-context.org= first +2. Run post-reboot verification: + - =uname -r= → confirm new kernel is running + - =systemctl --failed= → confirm zero failed services + - =journalctl -p err -b= → count and compare to pre-reboot (sharp drop = transient warnings were real, accumulated noise) +3. Re-evaluate the deferred warnings — most are gone without action +4. Full Phase 2 treatment for any survivors +5. Complete Phase 4, then wrap up (writes session entry to notes.org covering both halves as one unified session, spanning the reboot) +6. Delete =session-context.org= only after wrap-up is committed + +*Rationale:* Trying to cram the whole flow into one session risks either (a) wasting time investigating transient warnings that would have self-healed, or (b) losing context if a crash happens during the reboot gap. The two-stage pattern is explicit about the pause and survives interruptions. + +** Phase 4: Resolve .pacnew Files + +After updates, check for and resolve .pacnew files: + +#+begin_src bash +find /etc -name "*.pacnew" 2>/dev/null +#+end_src + +For each .pacnew file: +1. Diff the current config against the .pacnew version +2. Determine if the current config is actively managed (e.g., by Reflector, or manually customized) +3. If the current config is correct and the .pacnew adds nothing useful → delete the .pacnew +4. If the .pacnew has meaningful changes → merge them into the current config, then delete the .pacnew +5. Always explain the diff to Craig before deleting + +Common .pacnew files on ratio: +- =/etc/pacman.d/mirrorlist.pacnew= — safe to delete; mirrorlist is managed by Reflector +- =/etc/locale.gen.pacnew= — safe to delete if =en_US.UTF-8= is already uncommented + +* Health Checks Reference + +Run these checks in order. Each check produces one or more findings for the report. + +** 1. System Basics + +#+begin_src bash +echo "Date: $(date)" +echo "Hostname: $(hostnamectl hostname)" +echo "Kernel: $(uname -r)" +echo "Uptime: $(uptime -p)" +#+end_src + +Severity: INFO (always). + +** 2. Failed systemd Units + +#+begin_src bash +systemctl --failed +#+end_src + +- Any failed units → WARNING (CRITICAL if essential service like NetworkManager, fail2ban, cronie) +- No failed units → INFO "All units healthy" + +** 3. Journal Errors (Current Boot) + +#+begin_src bash +# Get total error count first +journalctl -p err -b --no-pager | wc -l + +# Then separate known-noisy services from real errors +journalctl -p err -b --no-pager | grep -v "Hands-Free Voice gateway" | tail -50 +#+end_src + +Known noisy patterns to filter when counting: +- bluetoothd "Hands-Free Voice gateway" — paired device out of range, polls every 60s +- pixman "_pixman_log_error" — benign Hyprland rendering debug message +- xkbcomp "Errors from xkbcomp are not fatal" — X compatibility noise + +When bluetooth spam is found, identify the device with =bluetoothctl devices Paired= and offer to remove stale pairings. + +- Recurring or service-impacting errors → WARNING +- Hardware errors (MCE, disk I/O) → CRITICAL +- Noise-level errors (known benign) → skip, but note the noise source + +** 4. Kernel & Hardware Events (journalctl -k) + +Check #3 catches error-priority journal messages, but many hardware issues show up as +patterns of lower-severity kernel messages (e.g., repeated USB disconnects, I/O retries). +This check scans the kernel log for those patterns. + +*Why =journalctl -k -b= and not =dmesg=:* on systems that suspend (laptops especially), =dmesg -T= computes wall-clock timestamps using =current_realtime − CLOCK_MONOTONIC=, but kernel printk timestamps include suspend time. After a suspend-heavy uptime, dmesg displays timestamps drifted forward by the cumulative suspend duration (e.g., May 17 displayed for messages logged on May 10). =journalctl -k= stamps CLOCK_REALTIME at message-write time, so dates are always correct. Performance difference is ~100 ms vs ~10 ms — negligible at this cadence. (See the 2026-05-10 entry in the Known Issues Log.) + +#+begin_src bash +# USB disconnect/reconnect cycles (repeated = hardware or power management issue) +journalctl -k -b --no-pager | grep -c "USB disconnect" + +# If disconnects found, show the devices and timestamps +journalctl -k -b --no-pager | grep "USB disconnect" | tail -20 + +# USB protocol errors (error -71 = EPROTO, error -110 = timeout) +journalctl -k -b --no-pager | grep -iE "usbhid.*error|usb.*error -[0-9]+" | tail -10 + +# I/O errors on block devices +journalctl -k -b --no-pager | grep -iE "I/O error|blk_update_request|Buffer I/O error" | tail -10 + +# PCIe errors +journalctl -k -b --no-pager | grep -iE "pcie.*error|AER|corrected error|uncorrected error" | tail -10 + +# Machine Check Exceptions (CPU hardware errors) +journalctl -k -b --no-pager | grep -iE "mce:|hardware error" | tail -10 + +# GPU faults or resets +journalctl -k -b --no-pager | grep -iE "amdgpu.*error|amdgpu.*fault|gpu reset|gpu hang" | tail -10 + +# Thermal throttling +journalctl -k -b --no-pager | grep -iE "thermal.*throttl|cpu.*throttl|prochot" | tail -10 + +# ACPI errors (power management issues) +journalctl -k -b --no-pager | grep -iE "acpi.*error|acpi.*warning" | tail -10 + +# USB autosuspend state on internal hubs (Framework 16 specific) +for d in /sys/bus/usb/devices/1-[234]; do + if [ -d "$d" ]; then + echo "$d: $(cat $d/product 2>/dev/null) | control=$(cat $d/power/control) | delay=$(cat $d/power/autosuspend_delay_ms)" + fi +done +#+end_src + +What to look for: +- *USB disconnects*: A few at boot or after resume is normal. >5 during a session, or repeated disconnect/reconnect of the same device = problem. On Framework 16, the keyboard and numpad modules are internal USB — disconnects here mean input loss. +- *Error -71 (EPROTO)*: USB protocol error, often caused by aggressive hub autosuspend. Check hub power settings. +- *I/O errors*: Any I/O error on NVMe → immediate CRITICAL. +- *MCE/hardware errors*: Always CRITICAL. +- *GPU faults*: Occasional corrected errors are INFO. Resets or hangs are WARNING. +- *Thermal throttling*: WARNING if active. Check if cooling is obstructed. +- *USB hub autosuspend*: Internal hubs with =control=auto= and =delay=0= → WARNING (causes input disconnects on Framework 16). + +- Repeated USB disconnects of input devices → WARNING +- USB protocol errors (EPROTO) → WARNING +- I/O errors, MCE, or hardware errors → CRITICAL +- GPU resets or hangs → WARNING +- Thermal throttling active → WARNING +- Aggressive hub autosuspend on internal devices → WARNING +- Clean dmesg → INFO + +** 5. Disk Health (SMART) + +#+begin_src bash +sudo smartctl -H /dev/nvme0n1 +sudo smartctl -H /dev/nvme1n1 +sudo smartctl -A /dev/nvme0n1 | grep -i "critical\|temperature\|wear\|error" +sudo smartctl -A /dev/nvme1n1 | grep -i "critical\|temperature\|wear\|error" +#+end_src + +- SMART overall health != PASSED → CRITICAL +- NVMe wear > =storage.smart_wear_warn_pct= or temp > =storage.smart_temp_warn_c= (TOML; seeded 80% / 70°C) → WARNING +- All healthy → INFO + +** 6. Disk Usage + +Dispatched by filesystem. Run every sub-check for which the capability is present — a host with both Btrfs and ZFS (hypothetical: ratio with a ZFS backup pool) runs 6a AND 6b. + +*** 6a. Btrfs Usage + +*Applies when:* =has_btrfs > 0= + +#+begin_src bash +sudo btrfs filesystem usage / +/usr/bin/df -h / /boot 2>/dev/null +#+end_src + +Btrfs reports two numbers that can look confusing: +- *Data allocated %* (e.g., 99.43%) — how full the allocated data chunks are. Btrfs allocates in chunks and can allocate more from the unallocated pool. +- *df %* (e.g., 69%) — actual used space vs total device size. This is what matters. + +Check the "Device unallocated" line — as long as there's unallocated space, Btrfs can grow its data chunks. + +Severity rules: +- df > =storage.df_crit_pct= (TOML; seeded 90%) → CRITICAL +- df > =storage.df_warn_pct= (seeded 80%) → WARNING +- df under the warn line → INFO (report both df% and unallocated space) + +*** 6b. ZFS Pool Health + +*Applies when:* =has_zfs > 0= + +#+begin_src bash +# Quick health summary across all pools +sudo zpool status -x # expect "all pools are healthy" + +# Full detail — state, errors, scrub, resilver +sudo zpool status + +# Capacity, fragmentation, health per pool +sudo zpool list -o name,size,alloc,free,capacity,health,frag + +# Per-dataset usage (top-level only; -d1 depth) +sudo zfs list -o name,used,avail,refer -s used +#+end_src + +Severity rules: +- Pool state != ONLINE → CRITICAL +- Any read/write/cksum error > 0 → CRITICAL +- Capacity > =storage.zfs_capacity_crit_pct= (TOML; seeded 90%) → CRITICAL +- Capacity > =storage.zfs_capacity_warn_pct= (seeded 80%) → WARNING (ZFS performance degrades) +- Scrub age > =storage.zfs_scrub_crit_days= (seeded 60) → CRITICAL +- Scrub age > =storage.zfs_scrub_warn_days= (seeded 35) → WARNING +- Resilver in progress → INFO (but prominent in the report — mention drive being replaced) +- Fragmentation > =storage.zfs_frag_info_pct= (seeded 50%) → INFO + +Parse scrub age from =zpool status= output: look for the =scan:= line — either =scrub repaired … in … with … errors on <date>= (completed) or =scrub in progress …= (running). + +*** 6c. Generic filesystem fallback + +*Applies when:* =root_fs= is ext4, xfs, or any other non-Btrfs non-ZFS filesystem + +#+begin_src bash +/usr/bin/df -h / /home /boot 2>/dev/null +#+end_src + +Severity rules: +- > =storage.df_crit_pct= on any mounted partition → CRITICAL +- > =storage.df_warn_pct= → WARNING +- Normal → INFO + +** 7. Snapshots + +Dispatched by snapshot tool. + +*** 7a. Snapper (Btrfs) + +*Applies when:* =snapshot = snapper= + +#+begin_src bash +snapper -c home list --columns number,date,description | head -5 +snapper -c home list --columns number,date,description | tail -5 +snapper -c home list | wc -l +snapper -c home get-config | grep -iE "cleanup|TIMELINE_LIMIT|NUMBER_LIMIT" +# oldest snapshot age (pile-up signal) +oldest=$(sudo ls /home/.snapshots/ 2>/dev/null | grep -E '^[0-9]+$' | sort -n | head -1) +[ -n "$oldest" ] && echo "oldest: $(sudo grep -oP '(?<=<date>).*(?=</date>)' "/home/.snapshots/$oldest/info.xml" 2>/dev/null | head -1)" +#+end_src + +Severity rules: +- Zero snapshots → WARNING +- Cleanup disabled → WARNING +- Snapshot count very high, OR oldest snapshot far older than intended retention (e.g. months back) → WARNING (pile-up) +- =TIMELINE_LIMIT_MONTHLY= / =_YEARLY= set high (e.g. 10) on a large, churny subvolume → WARNING: that keeps ~10 months of monthly snapshots, which silently hoard space (this bit /home on 2026-05-26 — monthly snapshots back to Feb were the top space holders). Sane values are the TOML's =[snapshots]= =timeline_*= keys (seeded HOURLY 6, DAILY 7, WEEKLY 2, MONTHLY 2, QUARTERLY 0, YEARLY 0). Manual (=single=) snapshots are NOT touched by timeline cleanup — delete stale ones explicitly. +- Normal range → INFO with count and date range + +*Disk-space deep-dive (on demand, only when /home is actually filling).* Per-snapshot reclaimable space is invisible without btrfs quotas. TEMPORARILY enable quota, rank by exclusive space, then DISABLE it again — quotas slow every snapshot deletion, so never leave them on: +#+begin_src bash +sudo btrfs quota enable /home && sudo btrfs quota rescan -w /home +sudo btrfs qgroup show -re --sort=-excl /home | head -20 # top exclusive (reclaimable) holders +sudo btrfs quota disable /home +#+end_src +Reality check: data deleted from the live fs but still in many old snapshots shows ~0 *exclusive* per snapshot (it's shared across them), and only frees once the whole chain holding it is pruned. So reclaiming a large deletion means pruning the old snapshots, not just the newest one. + +*** 7b. ZFS Snapshots + +*Applies when:* =snapshot = zfs-native= or =sanoid= + +#+begin_src bash +# All snapshots, newest first +sudo zfs list -t snapshot -o name,creation,used -s creation | tail -20 + +# Count per pool +sudo zfs list -t snapshot -H -o name | awk -F'@' '{print $1}' | sort | uniq -c + +# Oldest snapshot age per dataset (detect runaway retention) +sudo zfs list -t snapshot -H -o name,creation -s creation | head -20 + +# Auto-snapshot service status (zfs-auto-snapshot / sanoid / zrepl) +systemctl list-timers --all 2>/dev/null | grep -Ei "zfs|sanoid|zrepl" || echo "no auto-snapshot timer" +#+end_src + +Severity rules: +- Zero snapshots on a dataset that should have them → WARNING +- Snapshot count on any dataset > =snapshots.zfs_count_warn= (seeded 1000) → WARNING (runaway retention) +- Snapshot space used > =snapshots.zfs_space_warn_pct= (seeded 20%) of pool capacity → WARNING (heavy divergence) +- Auto-snapshot timer not running → WARNING +- Normal → INFO (counts per pool, oldest snapshot age) + +*** 7c. No snapshot system + +*Applies when:* =snapshot = none= + +Report =INFO: no snapshot system configured on this host= and skip. + +** 8. Memory and Swap + +#+begin_src bash +free -h +journalctl -b --no-pager | grep -i "out of memory\|oom-kill\|killed process" | tail -10 +#+end_src + +Ratio has 128GB RAM and no swap configured. No swap is expected — don't flag it. + +- OOM kills found → WARNING +- Normal → INFO + +** 9. CPU Temperatures + +#+begin_src bash +sensors 2>/dev/null || echo "lm_sensors not installed or not configured" +#+end_src + +Key sensors on ratio: +- =k10temp= → CPU (Tctl) +- =amdgpu= → GPU edge temp +- =nvme= → NVMe drives (2 drives) +- =cros_ec= → mainboard (power, memory, ambient, CPU) + +- CPU temp > =power.cpu_temp_warn_c= or GPU temp > =power.gpu_temp_warn_c= (TOML; seeded 90/95°C) → WARNING, CRITICAL if sustained +- Normal → INFO (report CPU and GPU temps) + +** 10. NTP Sync + +#+begin_src bash +chronyc tracking +#+end_src + +Ratio uses chrony for NTP. + +- Clock not synchronized → WARNING +- NTP service inactive → WARNING +- Last offset > =network.ntp_offset_warn_ms= (TOML; seeded 100 ms) → WARNING +- Synchronized → INFO + +** 11. rsyncshot Backups + +*Applies when:* =backup = source= + +#+begin_src bash +tail -30 /var/log/rsyncshot.log 2>/dev/null +#+end_src + +rsyncshot runs via root crontab: +- Hourly: =30 0-1,3-23 * * *= (every hour except 2:30) +- Daily: =30 2 * * *= (2:30 AM) + +Backups go to TrueNAS via rsync. Look for "rsyncshot completed successfully" and check the timestamp. + +- No daily backup in =backups.rsyncshot_daily_crit_hours= (seeded 48) → CRITICAL +- No hourly backup in =backups.rsyncshot_hourly_warn_hours= (seeded 3) → WARNING (only on hosts whose log shows hourly runs) +- Errors in log → WARNING +- Recent and healthy → INFO + +** 12. Tailscale + +*Applies when:* =mesh = tailscale= + +#+begin_src bash +tailscale status +#+end_src + +Expected peers: ratio, cjennings, truenas, velox, worker. pixel6 is a phone and is often offline — don't flag it. + +- Tailscale not running → WARNING +- Server peers (truenas, velox, worker) offline → WARNING +- All expected peers online → INFO + +** 13. fail2ban + +#+begin_src bash +sudo fail2ban-client status +sudo fail2ban-client status sshd 2>/dev/null +#+end_src + +- fail2ban not running → WARNING +- Running → INFO (note any recent bans for awareness) + +** 14. Package Maintenance + +Dispatched by package manager. + +*** 14a. pacman (Arch) + +*Applies when:* =pm = pacman= + +#+begin_src bash +# Orphaned packages +pacman -Qtdq 2>/dev/null + +# Pending updates +checkupdates 2>/dev/null + +# Package cache timer + current size +systemctl list-timers paccache.timer --no-pager +du -sh /var/cache/pacman/pkg 2>/dev/null + +# .pacnew files +find /etc -name "*.pacnew" 2>/dev/null +#+end_src + +Notes: +- =paccache.timer= runs weekly keeping 3 versions — fine for routine pruning, but it only trims *old versions*. A cache can still balloon from *breadth* (many packages, ≤3 versions each), where =paccache -r= finds nothing to prune. On 2026-05-26 the cache was 17 GB and =paccache -r= pruned 0; the space came from deeper levers: + - =sudo paccache -ruk0= — remove cache for *uninstalled* packages (safe, re-downloadable). + - =sudo paccache -rk1= — keep only 1 version of installed packages (frees the most; less downgrade headroom). + Together those took 17 GB → 5.8 GB. +- When removing orphans, pass package names as arguments (don't pipe through stdin — breaks snap-pac's snapper hook). +- Review orphans before removing — some (like =rust=) warrant discussion (Craig switched to =rustup=). + +Severity rules: +- Orphaned packages beyond the curated =curation.kept_orphans= set, in bulk (> ~20) → WARNING +- Pending updates > =updates.pending_warn= (TOML; seeded 50) → WARNING +- Unreviewed .pacnew files → WARNING +- Package cache > =packages.cache_warn_gb= (TOML; seeded 10 GB) → INFO: suggest =paccache -ruk0= (uninstalled) and/or =paccache -rk1= (keep 1) for a deeper reclaim beyond the weekly keep-3. +- Normal counts → INFO + +*** 14b. apt (Debian/Ubuntu/Mint) + +*Applies when:* =pm = apt= + +#+begin_src bash +# Pending updates +apt list --upgradable 2>/dev/null | tail -n +2 + +# Auto-removable packages +apt -s autoremove 2>&1 | grep -E "^Remv|packages will be REMOVED" + +# .dpkg-dist / .dpkg-new files (the apt equivalent of .pacnew) +find /etc -name "*.dpkg-dist" -o -name "*.dpkg-new" 2>/dev/null + +# unattended-upgrades timer +systemctl list-timers apt-daily.timer apt-daily-upgrade.timer --no-pager 2>/dev/null +#+end_src + +Severity rules: +- Pending security updates > 0 → WARNING +- Auto-removable packages > 30 → WARNING +- Unreviewed .dpkg-dist files → WARNING +- Normal → INFO + +*** 14c. none (TrueNAS SCALE, other appliances) + +*Applies when:* =pm = none= + +Report =N/A — package maintenance handled by middleware= and skip. On TrueNAS SCALE, OS updates go through the TrueNAS UI (System → Update); user-level =apt= exists but is unsupported and shouldn't be used. + +*** 14d. Orchestrator (topgrade) + +*Applies when:* =orch = topgrade= + +If topgrade is present, Phase 3 uses it rather than the raw package manager. Phase 1 still runs 14a/14b/14c above — topgrade is for the update run, not the status scan. + +** 15. App Logs (~/.local/var/log) + +#+begin_src bash +for log in ~/.local/var/log/*.log; do + if [ -f "$log" ]; then + errors=$(grep -ci "error\|critical\|fatal" "$log" 2>/dev/null) + if [ "$errors" -gt 0 ]; then + echo "$(basename $log): $errors error lines" + grep -i "error\|critical\|fatal" "$log" | tail -3 + echo "" + fi + fi +done +#+end_src + +Known noise patterns: +- *waybar*: LIBDBUSMENU-GLIB-WARNING "Unable to replace properties on 0" — caused by insync and zoom tray icons with broken dbusmenu implementations. Harmless. Filtered in waybar's log output via =grep -v= in hyprland.conf. +- *hyprland*: "_pixman_log_error" and "xkbcomp" — benign +- *gammastep*: "Wayland connection experienced a fatal error" — happens on session restart, self-recovers +- *dunst*: gdk_pixbuf assertion failures — usually a malformed notification icon, one-off + +Look for patterns that are NOT in the known noise list. + +- Coredumps within =logs.coredump_window_days= (TOML; seeded 14 — =coredumpctl list= and age-filter) → WARNING +- Recurring non-noise errors → WARNING +- Only known noise → INFO + +** 16. System Logs (/var/log) + +#+begin_src bash +# Recent pacman errors +grep -i "error\|failed" /var/log/pacman.log 2>/dev/null | tail -10 + +# fail2ban anomalies +tail -50 /var/log/fail2ban.log 2>/dev/null | grep -i "error\|warning" +#+end_src + +- Failed pacman transactions → WARNING +- fail2ban errors → WARNING +- Clean → INFO + +** 17. Docker/Podman + +*Applies when:* =virt= contains =docker= or =podman= + +#+begin_src bash +docker ps -a --filter "status=exited" --format "{{.Names}}: exited {{.Status}}" 2>/dev/null +docker system df 2>/dev/null +podman ps -a --filter "status=exited" --format "{{.Names}}: exited {{.Status}}" 2>/dev/null +podman system df 2>/dev/null +#+end_src + +The WinVM podman container is run on-demand and will often show as exited — this is expected. + +*Severity thresholds:* +- Stopped containers that should be running → WARNING +- Docker/Podman reclaimable > =services.docker_reclaim_warn_gb= *or* > =services.docker_reclaim_warn_pct= of total (TOML; seeded 5 GB / 50%) → WARNING (propose prune tiers in Phase 2) +- Docker/Podman reclaimable 1–5 GB → INFO (note for awareness) +- Expected stopped containers → INFO + +*Prune tiers to propose when reclaimable crosses the WARNING threshold:* +1. =docker container prune= — removes only stopped containers (~tens of MB, unlocks referenced images) +2. =docker image prune -a= — removes all images not used by any container (biggest bite; requires tier 1 first) +3. =docker system prune -a --volumes= — full nuke, includes unused volumes and networks. Present as the option when Craig wants to start fresh. + +Confirm with Craig before running any tier — especially if work-related containers (e.g. =deepsat-*=) are present. + +** 18. libvirt / VMs + +*Applies when:* =virt= contains =libvirt= + +#+begin_src bash +sudo virsh list --all 2>/dev/null || echo "libvirt not available" +#+end_src + +VMs (ultmos-15, etc.) are usually shut off unless Craig is actively using them. "shut off" is the normal state. + +- VMs in unexpected state → INFO +- libvirt not running → INFO + +** 19. Cron Jobs + +#+begin_src bash +systemctl is-active cronie +crontab -l 2>/dev/null | grep -v "^#" | grep -v "^$" +sudo crontab -l 2>/dev/null | grep -v "^#" | grep -v "^$" +#+end_src + +Expected user crontab entries: +- =0 12 * * *= log-cleanup + +Expected root crontab entries: +- =30 0-1,3-23 * * *= rsyncshot hourly +- =30 2 * * *= rsyncshot daily + +- cronie not running → WARNING +- Expected crontab entries missing → WARNING +- Running with expected entries → INFO + +** 20. Log Cleanup Cron + +#+begin_src bash +find ~/.local/var/log -type f -name "*.log" -mtime +7 2>/dev/null | head -10 +find ~/.local/var/log -type f 2>/dev/null | wc -l +#+end_src + +The user crontab runs =~/.local/bin/cron/log-cleanup= daily at noon, which should keep logs to ~7 days. + +- Log files older than =logs.app_log_warn_days= (TOML; seeded 7) present → WARNING (cleanup cron may not be running) +- Within the retention window → INFO + +** 21. Network + +#+begin_src bash +# DNS resolution (dig may not be available — use ping as fallback) +ping -c1 -W2 archlinux.org 2>/dev/null && echo "DNS OK" || echo "DNS FAILED" + +# NetworkManager status +nmcli general status 2>/dev/null +#+end_src + +- DNS resolution fails → CRITICAL +- NetworkManager not running or disconnected → CRITICAL +- All healthy → INFO + +* Report Format + +Present findings as a ranked org-mode table, sorted by severity (CRITICAL first, then WARNING, then INFO): + +#+begin_example +| # | Severity | Category | Finding | +|---+----------+-------------+--------------------------------| +| 1 | CRITICAL | rsyncshot | Daily backup failed 2 days ago | +|---+----------+-------------+--------------------------------| +| 2 | WARNING | disk usage | / at 85% capacity | +|---+----------+-------------+--------------------------------| +| 3 | WARNING | packages | 47 orphaned packages | +|---+----------+-------------+--------------------------------| +| 4 | INFO | uptime | 12 days | +|---+----------+-------------+--------------------------------| +| 5 | INFO | disk health | Both NVMe drives PASSED | +|---+----------+-------------+--------------------------------| +| 6 | INFO | tailscale | 3/3 peers online | +|---+----------+-------------+--------------------------------| +#+end_example + +After the table, announce the investigation plan: + +- If CRITICAL or WARNING issues exist: "Starting with #1 — investigating [category]: [finding]..." +- If only INFO items: "No issues found. System looks healthy. Here are the details..." + +* Known Issues Log + +Each entry is scoped to one host (or =any=). When Phase 1 cross-references findings against this log, it matches on both the issue signature AND the host — an entry tagged =:host: mybitch= won't suppress a finding on ratio. For pre-existing entries without an explicit host tag, the hostname is inferred from the entry title where possible (e.g., "mybitch —"). New entries should include =:host: <name>= for clarity. + +** 2026-03-24: mybitch — USB autosuspend causing keyboard/numpad disconnects +:host: mybitch +- Framework Laptop 16 internal Genesys Logic USB hubs (05e3:0610) had =control=auto= and =delay=0= +- Caused repeated USB disconnects of keyboard module (1-4.2) and numpad module (1-3.2) +- Error -71 (EPROTO) on reconnection attempts +- Fix: udev rule at =/etc/udev/rules.d/99-framework-usb-hub-no-autosuspend.rules= +- Sets =power/control=on= for matching hubs, disabling autosuspend +- Added check #4 (Kernel & Hardware Events) to this workflow to catch similar issues in future + +** 2026-02-27: Bluetooth journal spam (1,006 errors/boot) +:host: ratio +- bluetoothd polling "Craig's Pixel Buds" (B8:7B:D4:19:A6:01) every 60s while out of range +- Fix: removed stale pairing with =bluetoothctl remove B8:7B:D4:19:A6:01= +- Note: Craig has recurring Pixel Buds pairing issues with this machine + +** 2026-02-27: hyprlock deprecated config options (v0.9.2) +:host: ratio +- =general:grace= and =general:no_fade_in= moved to CLI flags (=--grace=, =--no-fade-in=) +- =input-field:fail_transition= replaced by =animations= block (=inputFieldColors= animation) +- Fix: removed deprecated options, added =animations= block to hyprlock.conf +- Config at: =~/code/archsetup/dotfiles/hyprland/.config/hypr/hyprlock.conf= + +** 2026-02-27: dunst coredump +:host: ratio +- Segfault in libglycin/gdk_pixbuf — malformed notification icon +- One-off, no action taken. Monitor for recurrence. + +** 2026-02-27: Missing udev script for Logitech Brio +:host: ratio +- Udev rule references =~/.local/bin/logitech-brio-settings.sh= which is a symlink to archsetup dotfiles +- Fails at boot because udev runs before home is fully available +- Script works fine for hotplug after login +- Decision: leave as-is + +** 2026-02-27: waybar LIBDBUSMENU-GLIB warnings (~8,000/session) +:host: ratio +- Caused by insync and zoom tray icons with broken dbusmenu implementations +- Both return empty arrays for menu root item (ID 0) +- No fix available (insync has no option to disable tray icon) +- Mitigation: added =grep -v "LIBDBUSMENU-GLIB-WARNING"= filter to waybar log line in hyprland.conf +- Takes effect on next Hyprland start + +** 2026-02-27: Package cleanup +:host: ratio +- Removed 51 orphaned packages (build deps, Python dev tools) +- Swapped =rust= pacman package for =rustup= (toolchain manager) +- Added note to =~/code/archsetup/inbox/rustup.txt= + +** 2026-02-27: System update via topgrade +:host: ratio +- 66 packages updated including Mesa 25.3→26.0, Firefox/Thunderbird 148, linux-firmware, Signal 8.0 +- No manual intervention required (checked Arch news) +- Reboot recommended for firmware and Mesa changes + +** 2026-02-27: Resolved .pacnew files +:host: ratio +- =/etc/pacman.d/mirrorlist.pacnew= — stock mirrorlist, safe to delete (Reflector manages the active one) +- =/etc/locale.gen.pacnew= — only added commented =en_SE.UTF-8=, current config correct, deleted + +** 2026-04-19: proton.VPN.service transient failure after topgrade +:host: ratio +- Appeared as a =failed= unit immediately after =topgrade= bumped iproute2 from 6.19.0 → 7.0.0 mid-session (9-day uptime) +- Self-healed on reboot; no config change required +- Pattern: VPN daemon holds routing/ip-rule state referencing old iproute2 ABI, can't reconcile after live package swap +- Classification: KNOWN-TRANSIENT on iproute2 upgrade → DEFER-TO-POST-REBOOT per Phase 2 rules + +** 2026-04-19: Tailscale DNS reachability warning on long uptime +:host: ratio +- =tailscale status= footer: "Tailscale can't reach the configured DNS servers. Internet connectivity may be affected." +- Appeared on 9-day uptime; all core functions (name resolution, netcheck, tailnet routing) tested working +- Cleared on reboot — footer warning gone, no config change +- Pattern: tailscaled's internal DNS health check accumulates stale state on long uptime, particularly after a mid-session iproute2 transition +- Classification: KNOWN-TRANSIENT on long uptime / iproute2 upgrade → DEFER-TO-POST-REBOOT per Phase 2 rules + +** 2026-04-19: wpa_supplicant bgscan error spam volume scales with uptime +:host: ratio +- Error: =bgscan simple: Failed to enable signal strength monitoring= +- Volume pre-reboot (9-day uptime): 1,200+ instances in journal +- Volume post-reboot (fresh boot): 1 instance total +- Root cause: RSSI monitoring ioctl not implemented in mt7925e driver (Framework Desktop WiFi 7 card) — each retry or roam event re-triggers the failure +- 1–3 instances per boot is baseline noise; only triage if volume > ~50 AND not explainable by long uptime +- Classification: KNOWN (mt7925e driver limitation, no upstream fix tracked); annotate low-volume occurrences as =KNOWN — mt7925e baseline= + +** 2026-04-19: Orphan package cleanup +:host: ratio +- Removed 11 packages (8 orphans + 3 cascading python build-tool deps): cli11, electron34, lua-lpeg, minizip-ng, python-build, python-hatchling, python-installer, yarn + python-editables, python-pyproject-hooks, python-trove-classifiers +- Freed 280 MiB +- Future note: electron34 and yarn were previously flagged "ask first" due to possible AUR build-dep relevance — post-removal, no AUR rebuild has failed. Can remove in future without asking unless an active AUR build references them. + +** 2026-04-21: velox ZFS — systemd-tmpfiles "Protocol driver not attached" +:host: velox +- Symptom: =systemd-tmpfiles-setup.service= (boot) and/or =systemd-tmpfiles-clean.service= (periodic) produce 10-30 =statx(...) failed: Protocol driver not attached= journal errors per run +- Root cause: on ZFS, statx against another service's =/var/tmp/systemd-private-*/tmp= mount returns errno 132 (ENOTNAM); ext4/btrfs don't surface this as an error. Bare systemd-tmpfiles unit has no =PrivateTmp= set, so it traverses sibling namespaces +- Fix: drop-in with =PrivateTmp=yes= at =/etc/systemd/system/systemd-tmpfiles-clean.service.d/zfs-private-tmp.conf= AND =.../systemd-tmpfiles-setup.service.d/zfs-private-tmp.conf= +- Applies to any ZFS-on-root Arch host. Not needed on btrfs hosts. + +** 2026-04-19: Docker image bloat cleanup +:host: ratio +- Freed 15.3 GB via =docker system prune -a --volumes= +- Removed 10 images (including nvidia/cuda 12.4 devel, postgis, nginx, python-slim, node-alpine, nerdfonts/patcher) and 4 exited deepsat-* work containers from 9 days prior +- 130 MB orphan volume survived prune (anonymous, not caught by =--volumes=); ignored +- Future: Check #17 now promotes reclaimable > 5 GB OR > 50% to WARNING — this will catch bloat earlier without Craig having to notice it + +** 2026-05-10: WiFi powersave default caused variable WiFi latency on mybitch + ratio +:host: mybitch, ratio (velox naturally clean — defensive config added anyway) +- Symptom: variable LAN/WiFi latency manifesting as "slow-feeling internet". On mybitch (Christine's laptop, the original report): gateway ping avg 67.9 ms, max 173.9 ms, mdev 55.7 ms at 1-second cadence vs avg 14.5 ms / mdev 16.4 ms at 0.2-second cadence (faster cadence kept the card awake, masking the issue). On ratio (Arch desktop): same pattern, mdev 46.7 ms, max 183.5 ms. +- Root cause: NetworkManager defaults to =wifi.powersave = 3= (enable) when no override is configured. On Mint/Ubuntu, this is shipped explicitly as =/etc/NetworkManager/conf.d/default-wifi-powersave-on.conf=. On Arch, it's the upstream NetworkManager default (no shipped file, but same effective behavior). On bursty traffic (typical browsing), the WiFi card sleeps between bursts and the first packet of each new burst takes 50-150 ms to wake it — pages feel like they "stall" before loading. +- Fix: drop =/etc/NetworkManager/conf.d/wifi-powersave-off.conf= with =wifi.powersave = 2= (disable). NetworkManager merges conf.d alphabetically; =wifi-...= sorts after =default-...= and =dns.conf= so the override wins. Restart NetworkManager applies it. +- Verification: post-fix gateway ping (both mybitch and ratio) avg 12 ms, max 21-24 ms, mdev 3 ms — 15-18× improvement in jitter, 8-9× improvement in max latency. Throughput unchanged at the ISP/router ceiling (~85 Mbit/s down, 25 Mbit/s up — same on both clients, so that's the connection limit, not a per-host issue). +- Velox edge: velox's WiFi card defaults to powersave-off at the driver/kernel level even without an explicit NM config (different chipset behavior). Latency was already healthy. Defensive =wifi-powersave-off.conf= added anyway so the configuration is uniform across the homelab and protected against future NM-default or driver-behavior changes. +- Tradeoff: ~0.5-1 W more power draw — irrelevant on AC, slight battery hit on battery. All three hosts are mostly on AC. +- Coverage status as of this entry: + - mybitch (Mint): explicit override applied, was active issue → resolved + - ratio (Arch): explicit override applied, was active issue → resolved + - velox (Arch laptop): explicit override applied defensively, no observable issue + - truenas: not on WiFi, N/A +- Pattern note: any new NM-using host added to the homelab should get this override at provisioning time. The 2026-04-30 mybitch upgrade soak picked up the issue indirectly; future host adds should set this proactively. + +** 2026-05-10: mybitch — keyboard soak verified closed (2026-03-24 USB hub fix held over 9-day uptime) +:host: mybitch +- Reminder from 2026-04-30 (overdue day 10) was to verify the 2026-03-24 USB-autosuspend fix holds in routine use. +- Verification (2026-05-10, mybitch uptime 1w 2d 2h): all three Framework 16 internal hubs (1-2, 1-3, 1-4) show =control=on, delay=0= — udev rule active. Total USB error count this boot: *0*. Soak passed. +- Closes the 2026-04-30 reminder. The 2026-03-27 mybitch BIOS-side fixes (which were gated on this soak result) are now unblocked. + +** 2026-05-10: dmesg -T displays future-dated timestamps after suspend cycles +:host: any (laptops affected; desktops rarely suspend) +- Symptom: =dmesg -T= shows wall-clock timestamps drifted forward by the cumulative suspend duration since boot. Example: mybitch on 2026-05-10 with 9d 2h monotonic uptime displayed messages dated May 15-17 (~7 days of suspend during the boot session pushed the displayed dates 7 days into the future). +- Root cause: kernel printk timestamps include suspend time (boottime-style clock). =dmesg -T= computes the boot epoch using =current_realtime − CLOCK_MONOTONIC=, but =CLOCK_MONOTONIC= excludes suspend. The two clocks diverge by however long the system was suspended during the current boot, and =dmesg -T= adds the (boottime-correct) message offset to a (monotonic-derived) boot epoch — producing future-dated displays. +- Verification: =journalctl -k -b= shows the same kernel messages with correct timestamps because journald stamps =CLOCK_REALTIME= at message-write time. =date=, =timedatectl=, and =/proc/stat btime= are all correct. +- Workflow fix (applied 2026-05-10): Phase 1 check #4 now uses =journalctl -k -b= instead of =dmesg=. Tradeoff is ~100 ms vs ~10 ms per query — negligible at this cadence. Side benefits: cross-boot queries (=-b -1=, =-b -2=) and longer history retention than the kernel ring buffer. +- Live cleanup (optional): a reboot resets both clocks and =dmesg -T= comes back accurate until the next long suspend session. Not urgent — the workflow change routes around the bug regardless. +- Out of scope: the dmesg tool itself isn't going to change behavior here; this is a long-standing util-linux design decision around how to translate ring-buffer timestamps. We don't fight it; we use the right tool. + +** 2026-05-10: aardvark-dns "empty response" spam during WinVM podman runs +:host: ratio +- Pattern: =aardvark-dns[N]: <port> dns request got empty response= logged at error priority while the podman-rootless aardvark-dns daemon forwards DNS for the WinVM container +- Volume: ~1,200 lines per WinVM session (Windows guest aggressively retries DNS lookups, every retry that gets back NOERROR/empty triggers one line) +- Trigger: only logs while a podman container with the rootless aardvark-dns network is running. Daemon stops with the container, so journal noise is bounded by WinVM uptime +- Active when looked at: aardvark-dns daemon is not running between WinVM sessions; these errors are retrospective journal entries, not a live issue +- Classification: KNOWN — annotate as =KNOWN — WinVM podman aardvark-dns DNS retry noise= and check the surrounding podman/WinVM lifecycle to confirm correlation. Volume scales with WinVM session length. +- No fix in scope. Suppression options if it becomes annoying: (a) journald filter rule for unit pattern, (b) switch WinVM to a different DNS path. Neither pursued today. + +** 2026-05-10: cameractrls cameraptzmidi.py SEGV during Python 3.14 exit +:host: ratio +- =python3 /usr/lib/python3.14/site-packages/CameraCtrls/cameraptzmidi.py -l= prints its output (e.g. "JDS Labs Element IV MIDI 1:32:0") and then SEGVs during interpreter shutdown +- Stack trace lives entirely inside =libpython3.14.so= on the =Py_Exit= path — no cameractrls frames, no asound frames at the crash point +- Pattern: ctypes-loaded =libasound= atexit handler vs Python 3.14 finalization order; the script's job completes before the crash so functional behavior is fine +- Reproduces every invocation. Three coredumps on 2026-05-08 11:49–11:50 were Craig running it three times in a row, not a regression burst +- Classification: KNOWN (upstream Python 3.14 / cameractrls issue, no local fix). Annotate future cameraptzmidi coredumps as =KNOWN — Py_Exit / libasound finalization= +- Real fix is upstream: Python 3.14 ctypes-finalization change or cameractrls explicit asound cleanup before exit. Not worth tracking locally. + +** 2026-05-11: mybitch — hard freeze caused by amdgpu iGPU MES hang; mitigated with =amdgpu.cwsr_enable=0= +:host: mybitch +- Symptom: full system freeze during active GUI use (Christine typing in a browser). Desktop frozen, keyboard + trackpad dead, off the network. Not the s2idle USB-keyboard bug — the machine was awake, not resuming from suspend, and the WiFi (M.2 PCIe, not USB) was dead too, so a USB-hub-only failure is excluded. Required hard power-cycle. =last= records the prior session as ending in "crash". +- Evidence in =journalctl -b -1=: the journal *stops dead* with no panic / oops / hung-task warning / OOM / GPU-reset trace — classic hard hang (kernel wedged, logging stopped). pstore empty. Preceding the freeze: =amdgpu 0000:c5:00.0: amdgpu: MES failed to respond to msg=MISC (WAIT_REG_MEM)= + =failed to reg_write_reg_wait= — *38 occurrences* over the 10-day boot, *zero* in every prior boot including the pre-upgrade 6.8.0-110 stretch. Last one ~4.7 min before the hang. =c5:00.0= is the *integrated* GPU (Radeon 780M / Phoenix1, GFX 11.0.x); =03:00.0= is the dGPU (RX 7700S) — the dGPU's =PSP/SMU is resuming= log lines are routine runtime-PM, not errors. +- Root cause class: amdgpu MES (Micro Engine Scheduler) firmware going unresponsive on AMD GFX11 APUs (Phoenix 780M, Strix Point, Kraken Point) — a known issue across a wide range of recent kernels (6.11 → 6.18+), no clean upstream fix. The MES errors appeared exactly when mybitch moved to kernel 6.17.0-23 (Mint 22.3 HWE-edge, adopted 2026-04-30 to fix the s2idle USB-keyboard bug). Tracked at: Framework Community "AMD GPU MES Timeouts Causing System Hangs", ROCm issues #3207 / #5590 / #5844, drm/amd GitLab #2986. +- Mitigation applied 2026-05-11: =amdgpu.cwsr_enable=0= added to =GRUB_CMDLINE_LINUX_DEFAULT= in =/etc/default/grub= on mybitch (backup: =/etc/default/grub.bak-2026-05-11-amdgpu-cwsr=), =update-grub= run, param verified in all 6 grub.cfg entries. Disables Compute Wave Store and Resume — the feature several reporters found triggers the MES firmware hang. Narrow, low-risk. Takes effect on next reboot. +- Watch: after the next reboot, check =journalctl -k -b | grep -c 'MES failed'= — should stay 0. If a freeze recurs even with cwsr disabled, escalate to =amdgpu.mes=0= (disables hardware MES scheduling entirely, falls back to KIQ; heavier hammer). Last resort = pin back to 6.8.0-111 (still installed), but that reintroduces the s2idle USB-keyboard bug, so not clean. +- *Update 2026-05-12:* =cwsr_enable=0= did NOT reduce the MES errors — 14 "MES failed to respond" hits in the first 14.5h boot after the fix (≈0.97/h) vs. 38 over the prior 10-day boot (≈0.16/h), i.e. *higher* rate; no freeze yet. Escalated proactively (Christine about to travel with the laptop for a week): added =amdgpu.mes=0= to =/etc/default/grub= alongside =cwsr_enable=0= (backup =/etc/default/grub.bak-2026-05-12-amdgpu-mes=, =update-grub= run, in all 6 grub.cfg entries). Takes effect on the next reboot (after the in-progress first rsyncshot backup). =mes=0= disables the GFX11 hardware MES scheduler → kernel falls back to the mature legacy KIQ path → sidesteps the hanging MES firmware entirely. Tradeoff: KIQ-on-GFX11 is a less-traveled config (small chance of cosmetic display/modeset quirks; irrelevant for a light desktop workload). Post-reboot check: =journalctl -k -b | grep -c 'MES failed'= should be 0; watch for any new amdgpu display oddities. (Also tracked in =homelab-inventory/mybitch-laptop.org= → Operational Changes Log.) +- *Update 2026-05-12b (logged retroactively 2026-05-25):* =mes=0= was swapped for =uni_mes=0= the same day (backup =/etc/default/grub.bak-2026-05-12b-uni-mes=). This refinement was applied but never written into this entry until the 2026-05-25 health check found the live cmdline disagreeing with the documented =mes=0= mitigation. Current cmdline: =amdgpu.cwsr_enable=0 amdgpu.uni_mes=0=. +- *Update 2026-05-25 (health check):* Kernel bumped 6.17.0-23 → 6.17.0-29-generic (Mint HWE). MES errors persist: 49 =MES failed to respond= / =reg_write_reg_wait= on =c5:00.0= over a 3d5h boot (≈0.63/h), *no hard freeze this uptime*. Key finding from =modinfo amdgpu= on 6.17.0-29: =mes= now defaults to 0 (disabled) and =uni_mes= defaults to 1 (enabled) — between -23 and -29 AMD moved GFX11 onto the *unified* MES path, so =uni_mes=0= is the current-kernel equivalent of the old =mes=0= mitigation. Setting =amdgpu.mes=0= explicitly on this kernel just restates the default and won't change behavior; the errors are on the unified-MES path, which is already disabled. Decision (Craig, 2026-05-25): leave the cmdline as-is — there is no stronger *documented* knob left to pull beyond =uni_mes=0=, and there's no freeze to chase. The newer kernel appears to recover from each MES timeout rather than wedging. Watch freeze behavior; escalation lever if a hard freeze recurs would be the experimental =amdgpu.mes=0 amdgpu.mes_kiq=0 amdgpu.uni_mes=0= full-legacy-KIQ path (not applied — risky on a traveling laptop) or pinning back to 6.8.0-111 (reintroduces the s2idle USB-keyboard bug). +- Classification: when a future health check on mybitch sees =MES failed to respond= / =reg_write_reg_wait= on =c5:00.0=, annotate as =KNOWN — amdgpu GFX11 iGPU MES hang= and check whether the count is climbing despite the mitigations; a single hard freeze with the dead-journal signature is the same issue recurring. On kernel 6.17.0-29+ the box runs =uni_mes=0= (the unified-MES disable), which is the heaviest documented knob in use — MES errors persisting on it without a freeze is the expected steady state, not an escalation signal. A *hard freeze* with the dead-journal signature is the real escalation trigger; remaining levers at that point are the experimental full-legacy-KIQ cmdline or pinning back to 6.8.0-111. + +** 2026-05-11: mybitch — power-profiles-daemon was stuck on power-saver (now balanced); benign dGPU power-limit error +:host: mybitch +- Symptom: =amdgpu 0000:03:00.0: amdgpu: New power limit (30) is out of range [100,120]= + =amdgpu: Failed to set power limit value= at error priority, once per boot (and on profile changes). +- Root cause: =power-profiles-daemon= in the =power-saver= profile tried to cap the discrete RX 7700S (=03:00.0=) at 30W, but the GPU's settable power-cap range is [100W, 120W] (current cap 100W). amdgpu rejected the out-of-range request → cosmetic error; the dGPU stayed at its 100W floor regardless. Underlying issue: PPD persists the last-set profile in =/var/lib/power-profiles-daemon/state.ini= and restores it on every boot — someone had set =power-saver= long ago (state.ini mtime Oct 2025) and it had been pinned there ever since. Christine runs mybitch on AC a lot, so this wasn't what she wanted. +- Resolution (2026-05-11): set the profile to =balanced=. =powerprofilesctl set= over SSH is polkit-denied (=switch-profile= needs an active local session), so done by editing =state.ini= directly under sudo (=Profile=power-saver= → =Profile=balanced=) with =power-profiles-daemon= stopped, then restarting it. Verified =powerprofilesctl get= → =balanced=; persists across reboots; no new power-limit errors. Christine can still flip the profile anytime via the Cinnamon power applet. +- Classification: if the =New power limit (30) out of range= error reappears, it means the profile drifted back to =power-saver= — annotate as =KNOWN — PPD power-saver dGPU cap rejection= and re-set to =balanced= the same way. Harmless either way; the only reason to fix it is that power-saver throttles the box on AC. + +** 2026-05-18: velox — NVMe Error Information Log entries from kernel optional-feature probing +:host: velox +- Symptom: =smartctl -A /dev/nvme0n1= shows =Error Information Log Entries: 9,530= and climbing slowly over time, despite =Media and Data Integrity Errors: 0= and =SMART overall-health PASSED=. +- Detail (=smartctl -l error /dev/nvme0n1=): all 9,530 entries are a single error class — =Status 0x4004 "Invalid Field in Command"= on the Admin Submission Queue (SQId 0). One row in the error info log, accumulated over 20,079 power-on hours (~2.3 years of drive uptime). +- Root cause: kernel or userspace polls for an optional NVMe feature/log page the drive doesn't implement; the controller rejects with =Status 0x4004=, the host sees ENOTSUP and moves on, but the drive faithfully logs every rejection. Common on NVMe drives where newer kernels probe for features added after the drive's firmware was written. +- Classification: KNOWN — annotate future findings as =KNOWN — NVMe Invalid-Field-in-Command from kernel optional-feature probing=. The counter going up is not a failure signal; only =Media and Data Integrity Errors=, =Critical Warning=, or =Available Spare= changes indicate actual drive health regression. +- Out of scope: no kernel-side fix worth chasing. The probe is harmless and would have to be tracked down to a specific kernel subsystem; not worth the time when the only effect is a benign counter. + +** 2026-05-25: mybitch — timeshift SIGSEGV on =--check --scripted= cron runs (GObject teardown race) +:host: mybitch +- Symptom: =coredumpctl= shows intermittent SIGSEGV of =/usr/bin/timeshift= during the hourly =timeshift --check --scripted= cron job (e.g. 2026-05-23 02:00, 2026-05-24 23:00 — 2 in a 3-day window). Crash is at address =0x30= inside =libgobject-2.0.so.0= =g_object_unref=, with a second thread in =g_file_get_contents= — a multi-threaded GObject teardown race. +- Timing: the process runs ~29 s (started :01, segfault :30) — it completes its work and dies during cleanup/exit, the same "job done, crash on teardown" shape as the cameractrls Py_Exit case. +- Backups unaffected: hourly =backup.log= files in =/var/log/timeshift/= are present every hour, =--list= works, 50 snapshots on disk. timeshift rsyncs to a temp dir then promotes, so a crashed run leaves at worst an incomplete dir the next run ignores — not a corrupt snapshot. +- Long-standing, not a regression: an older coredump from 2026-01-06 has the identical two-thread =g_object_unref= / =g_file_get_contents= signature. Version: =timeshift 25.12.4+zena= (Mint's maintained fork); apt shows 0 pending so no newer build to move to. +- Classification: KNOWN — annotate future findings as =KNOWN — timeshift zena teardown SIGSEGV on --check=. A real escalation signal would be missing hourly =backup.log= entries or a gap in the snapshot timeline; the coredumps alone are cosmetic. No fix in scope (upstream zena-fork bug); suppression isn't worth it. + +** 2026-06-13: ratio — dockerized telega-server SIGSEGVs in musl build +:host: ratio +- Symptom: =coredumpctl= shows repeated SIGSEGVs from =telega-server -O 31 -l /home/cjennings/.telega/telega-server.log -v 3= running inside the Telega Docker container. Example stack is entirely in the container's musl loader plus =/usr/bin/telega-server=, not host kernel, GPU, storage, or memory paths. +- Evidence: =~/.telega/telega-server.log= ends with =Unexpected char 'm' in plist value= followed by =Assertion failed: false (telega-dat.c: tdat_plist_value: 500)=. Surrounding TDLib traffic includes sticker/custom-emoji metadata such as =documentAttributeCustomEmoji= and =PhotoSizeSourceThumbnail[Thumbnail, type = m]=. +- Prior local triage: =~/.emacs.d/todo.org= recorded the same issue on 2026-06-11 as spontaneous memory-corruption crashes in =zevlg/telega-server:latest='s musl build, with several coredumps occurring without action-verb traffic. Telega package installed at the 2026-06-13 health check was =20260513.509=; MELPA had =20260604.2321= available. +- Functional status at 2026-06-13 check: no coredumps yet that day; Telegram scans still worked from cached chat state. Treat as an app/server-container crash, not a machine-health fault. +- Classification: KNOWN — annotate future =telega-server= coredumps on ratio as =KNOWN — dockerized telega-server musl SIGSEGV= if the signature matches =tdat_plist_value= / unexpected plist value or otherwise stays inside the Telega container. Escalate only if crashes become continuous, break Telegram workflows, or appear after moving off the Docker musl build. +- Deferred remediation options, in order of least disruption: update the Emacs =telega= package, rebuild/pull a newer =telega-server= image, pin a known-good pre-2026-06 image digest, build =telega-server= natively, or report upstream with =coredumpctl= and log evidence. diff --git a/scripts/hypr-live-update-guard b/scripts/hypr-live-update-guard index 4f561ae..e78200d 100755 --- a/scripts/hypr-live-update-guard +++ b/scripts/hypr-live-update-guard @@ -3,15 +3,28 @@ # hypr-live-update-guard - abort a live GPU/compositor library upgrade. # # Installed as a pacman PreTransaction hook. When an upgrade transaction -# includes GPU/compositor runtime libraries (mesa, hyprland, wayland, GPU -# drivers, ...) AND a Hyprland session is running, this aborts the -# transaction BEFORE any package is swapped. Replacing those libraries out -# from under a live compositor makes the next GPU-lib call hit a now -# "(deleted)" file and SIGABRT, taking the Wayland clients down with it -# (hit on ratio 2026-06-07: mesa + hyprland upgraded live, Hyprland crashed -# and took awww/insync/emacs with it). Aborting at PreTransaction is the -# safe point: nothing has been replaced yet, so the running session is -# untouched and the user can re-run the upgrade from a TTY. +# would CHANGE the on-disk version of GPU/compositor runtime libraries +# (mesa, hyprland, wayland, GPU drivers, ...) AND a Hyprland session is +# running, this aborts the transaction BEFORE any package is swapped. +# Replacing those libraries out from under a live compositor makes the next +# GPU-lib call hit a now "(deleted)" file and SIGABRT, taking the Wayland +# clients down with it (hit on ratio 2026-06-07: mesa + hyprland upgraded +# live, Hyprland crashed and took awww/insync/emacs with it). Aborting at +# PreTransaction is the safe point: nothing has been replaced yet, so the +# running session is untouched and the user can re-run from a TTY. +# +# Version-aware (2026-07-08): a same-version reinstall writes identical +# bytes over identical bytes and is harmless to the live session — the +# maintenance console's integrity REINSTALL is exactly that after an +# update, and the name-only guard was blocking it. Each target's installed +# version (pacman -Q) is compared against the sync-db candidate +# (expac -S %v); targets that match are pure reinstalls and pass. Both are +# read-only queries and run fine inside a hook — the ALPM lock only gates +# other transactions (pinned live with db.lck present). A target whose +# versions can't be resolved (AUR package, a -U transaction installing a +# local file, expac absent) reads as unknown and blocks conservatively — +# a -U of a guard-listed package at repo-matching version would slip +# through, which is accepted as pathological. # # Pacman feeds the matched package names on stdin (NeedsTargets). # @@ -21,13 +34,23 @@ # HYPR_GUARD_SENTINEL path whose existence also proceeds anyway # (default /run/archsetup-allow-live-gpu-update, # cleared on reboot since /run is tmpfs) +# HYPR_GUARD_VERSIONS "pkg installed candidate" lines replacing the +# pacman/expac lookups; when set, a package absent +# from the map reads as unknown (blocks) set -u sentinel="${HYPR_GUARD_SENTINEL:-/run/archsetup-allow-live-gpu-update}" -# Explicit override: the user knows what they're doing. -if [ "${HYPR_ALLOW_LIVE_UPDATE:-0}" = "1" ] || [ -e "$sentinel" ]; then +# Explicit override: the user knows what they're doing. The sentinel is +# consumed as it's honored — one touch buys exactly one transaction, so a +# leftover from a crashed caller can't keep the guard disarmed until +# reboot. +if [ "${HYPR_ALLOW_LIVE_UPDATE:-0}" = "1" ]; then + exit 0 +fi +if [ -e "$sentinel" ]; then + rm -f "$sentinel" 2>/dev/null || true exit 0 fi @@ -43,17 +66,63 @@ hyprland_running() { # this is exactly the from-a-TTY-after-logout path the warning points to. hyprland_running || exit 0 -# Collect the triggering packages (stdin from NeedsTargets) for the message. -pkgs=$(cat 2>/dev/null | sort -u | tr '\n' ' ') +# Collect the triggering packages (stdin from NeedsTargets). +targets=$(cat 2>/dev/null | sort -u) + +# "installed candidate" for a package, or nothing when unknown. +versions_for() { + if [ -n "${HYPR_GUARD_VERSIONS+x}" ]; then + printf '%s\n' "$HYPR_GUARD_VERSIONS" \ + | awk -v p="$1" '$1 == p { print $2, $3; exit }' + else + inst=$(pacman -Q -- "$1" 2>/dev/null | awk '{ print $2 }') + # expac prints one line per repo carrying the package; the first is + # the repo pacman resolves from + cand=$(expac -S '%v' -- "$1" 2>/dev/null | head -n 1) + printf '%s %s\n' "$inst" "$cand" + fi +} + +# Split the targets into version-changing (dangerous live) and same-version +# reinstalls (harmless). Unknown versions count as dangerous. +changed="" +count=0 +for pkg in $targets; do + count=$((count + 1)) + # shellcheck disable=SC2046 # splitting "inst cand" into $1 $2 is the point + set -- $(versions_for "$pkg") + inst=${1:-} + cand=${2:-} + if [ -n "$inst" ] && [ -n "$cand" ] && [ "$inst" = "$cand" ]; then + continue # pure reinstall: identical bytes, no swap hazard + fi + changed="$changed $pkg" +done + +# Every guarded target is a pure reinstall -- nothing changes on disk that +# the live session has mapped. Let it through silently. +if [ "$count" -gt 0 ] && [ -z "$changed" ]; then + exit 0 +fi + +# An empty target list shouldn't normally happen (the hook only fires when +# dangerous targets exist); if Hyprland is up, stay safe and abort. +pkg_lines="" +for pkg in $changed; do + pkg_lines="${pkg_lines} - ${pkg} +" +done +[ -n "$pkg_lines" ] || pkg_lines=" (GPU/compositor runtime libraries) +" cat >&2 <<EOF ========================================================================== BLOCKED: live GPU/compositor library upgrade while Hyprland is running ========================================================================== - Packages in this upgrade can crash the running compositor if swapped now: - ${pkgs:-(GPU/compositor runtime libraries)} - + Packages in this upgrade change version and can crash the running + compositor if swapped now: +${pkg_lines} Replacing these out from under a live Hyprland session makes the next GPU-lib call hit a deleted library and SIGABRT, taking your Wayland apps down with it (and risking an unclean shutdown). diff --git a/scripts/testing/maint-scenarios/10-journal-vacuum.sh b/scripts/testing/maint-scenarios/10-journal-vacuum.sh new file mode 100644 index 0000000..b61de49 --- /dev/null +++ b/scripts/testing/maint-scenarios/10-journal-vacuum.sh @@ -0,0 +1,30 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: the journal vacuum executes against live journald at the TOML floor. +# +# What this proves: the TOML edit reaches the remedy's argv (vacuum target +# and warn threshold are the same key), journalctl --vacuum-size runs for +# real against live journald, and the metric grades ok against the edited +# floor afterwards. +# +# What it deliberately does NOT prove: size reduction. The key is int-GB and +# bottoms out at 1, vacuum only touches archived files, and a VM-sized +# journal never nears 1 GB — a genuine reduction test would need journald +# rate-limit surgery plus ~1 GB of spam per run. A pre/post usage comparison +# is also unsound here: journald keeps landing in-flight writes (the spam is +# async, and every ssh hop logs), so usage can grow across a correct vacuum. +SCENARIO_DESC="journal vacuum runs at the TOML size floor" +SCENARIO_GROUP="logs" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "sed -i 's/^journal_disk_warn_gb *=.*/journal_disk_warn_gb = 1/' /root/.config/archsetup/maintenance-thresholds.toml" \ + && mexec "for i in \$(seq 1 20); do head -c 900000 /dev/urandom | base64 | systemd-cat -t maint-scenario-spam; done; journalctl --rotate; journalctl --flush >/dev/null 2>&1 || true" +} + +scenario_fix() { + mfix journal_vacuum +} + +scenario_assert() { + massert_metric journal_disk ok +} diff --git a/scripts/testing/maint-scenarios/11-coredump-age-out.sh b/scripts/testing/maint-scenarios/11-coredump-age-out.sh new file mode 100644 index 0000000..06d6fa0 --- /dev/null +++ b/scripts/testing/maint-scenarios/11-coredump-age-out.sh @@ -0,0 +1,21 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: coredump files beyond the forensic window are aged out. +# +# coredumpctl has no clean verb, so the remedy is a find -mtime +N -delete +# over /var/lib/systemd/coredump. Break plants one file older than the +# window and one fresh file; assert checks exactly the old one is gone. +SCENARIO_DESC="aged coredump files removed, fresh ones kept" +SCENARIO_GROUP="logs" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "mkdir -p /var/lib/systemd/coredump && touch -d '30 days ago' '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && touch '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'" +} + +scenario_fix() { + mfix coredump_clean +} + +scenario_assert() { + mexec "test ! -e '/var/lib/systemd/coredump/core.maintscenario-old.0.zst' && test -e '/var/lib/systemd/coredump/core.maintscenario-new.0.zst'" +} diff --git a/scripts/testing/maint-scenarios/20-cache-keep3.sh b/scripts/testing/maint-scenarios/20-cache-keep3.sh new file mode 100644 index 0000000..a9a6ee5 --- /dev/null +++ b/scripts/testing/maint-scenarios/20-cache-keep3.sh @@ -0,0 +1,24 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: paccache -rk3 trims an installed package's cache to 3 versions. +# +# paccache decides from filenames, so five fake versions of a package that +# IS installed (bash) stand in for a fat cache. The bootstrap may leave a +# real bash package in the cache too, so the assert uses the ordering +# invariant rather than a count: the two newest fakes survive any keep-3 +# window that contains them, and the two oldest fall outside it whether or +# not a real (newer) version is cached alongside. +SCENARIO_DESC="pacman cache trimmed to three versions per package" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "for v in 1.0.1 1.0.2 1.0.3 1.0.4 1.0.5; do touch /var/cache/pacman/pkg/bash-\$v-1-x86_64.pkg.tar.zst; done; ls /var/cache/pacman/pkg/ | grep -c '^bash-'" +} + +scenario_fix() { + mfix cache_clean +} + +scenario_assert() { + mexec "test -e /var/cache/pacman/pkg/bash-1.0.5-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-1.0.4-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.2-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/bash-1.0.1-1-x86_64.pkg.tar.zst" +} diff --git a/scripts/testing/maint-scenarios/21-cache-uninstalled.sh b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh new file mode 100644 index 0000000..ea788e8 --- /dev/null +++ b/scripts/testing/maint-scenarios/21-cache-uninstalled.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: paccache -ruk0 drops every cached version of uninstalled packages. +# +# Fake versions of a package that is NOT installed disappear entirely, while +# an installed package's cached version survives the uninstalled-only pass. +SCENARIO_DESC="cached versions of uninstalled packages dropped" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "touch /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst" +} + +scenario_fix() { + mfix cache_clean_uninstalled +} + +scenario_assert() { + mexec "test ! -e /var/cache/pacman/pkg/maintghostpkg-2.1-1-x86_64.pkg.tar.zst && test ! -e /var/cache/pacman/pkg/maintghostpkg-2.2-1-x86_64.pkg.tar.zst && test -e /var/cache/pacman/pkg/bash-9.9-1-x86_64.pkg.tar.zst" +} diff --git a/scripts/testing/maint-scenarios/22-orphan-remove.sh b/scripts/testing/maint-scenarios/22-orphan-remove.sh new file mode 100644 index 0000000..8412128 --- /dev/null +++ b/scripts/testing/maint-scenarios/22-orphan-remove.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a real orphan is created, detected, and removed by name. +# +# Installing `tree` --asdeps with no dependents makes it a true orphan +# (pacman -Qtdq lists it). The remedy is item-based: maint fix orphan_remove +# tree. Assert covers both the package state and the metric verdict. +SCENARIO_DESC="orphan package removed via the item-based remedy" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "pacman -S --noconfirm --asdeps tree && pacman -Qtdq | grep -qx tree" +} + +scenario_fix() { + mfix orphan_remove tree +} + +scenario_assert() { + mexec "! pacman -Qi tree >/dev/null 2>&1 && ! pacman -Qtdq | grep -qx tree" \ + && massert_metric pkg_orphans ok +} diff --git a/scripts/testing/maint-scenarios/23-pacnew-delete.sh b/scripts/testing/maint-scenarios/23-pacnew-delete.sh new file mode 100644 index 0000000..23cd8d1 --- /dev/null +++ b/scripts/testing/maint-scenarios/23-pacnew-delete.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a .pacnew file is deleted through the validated priv verb. +# +# pacdiff --output finds the planted file; the remedy takes the path as its +# item and rm's it. A neighbouring non-pacnew file proves the verb's path +# validation keeps the blast radius to the named file. +SCENARIO_DESC="planted .pacnew removed by path" +SCENARIO_GROUP="packages" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "cp /etc/pacman.conf /etc/pacman.conf.pacnew && pacdiff --output | grep -q pacman.conf.pacnew" +} + +scenario_fix() { + mfix pacnew_delete /etc/pacman.conf.pacnew +} + +scenario_assert() { + mexec "test ! -e /etc/pacman.conf.pacnew && test -e /etc/pacman.conf" \ + && massert_metric pkg_pacnew ok +} diff --git a/scripts/testing/maint-scenarios/30-unit-reset.sh b/scripts/testing/maint-scenarios/30-unit-reset.sh new file mode 100644 index 0000000..07eed53 --- /dev/null +++ b/scripts/testing/maint-scenarios/30-unit-reset.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: a failed unit's state is reset. +# +# A transient oneshot that exits 1 lands in systemctl --failed; the remedy +# resets it and the failed list returns to its pre-break size. The unit name +# rides the remedy's item argument, exercising the priv verb's unit-name +# validation. The base image carries its own failed unit (grub-btrfsd — no +# snapshot dirs in the VM), so the assert scopes to this scenario's unit and +# the count delta, never the absolute failed list. +SCENARIO_DESC="failed unit reset clears the failed list" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="any" + +scenario_break() { + PRE_FAILED_COUNT=$(mexec "systemctl --failed --no-legend | wc -l") \ + && mexec "systemd-run --unit=maint-scenario-fail --service-type=oneshot --no-block /bin/false; sleep 2; systemctl is-failed maint-scenario-fail.service" +} + +scenario_fix() { + mfix unit_reset maint-scenario-fail.service +} + +scenario_assert() { + local pre="$PRE_FAILED_COUNT" + unset PRE_FAILED_COUNT # don't leak scenario state past this scenario + mexec "! systemctl is-failed maint-scenario-fail.service >/dev/null 2>&1" \ + && [ "$(mexec "systemctl --failed --no-legend | wc -l")" = "$pre" ] +} diff --git a/scripts/testing/maint-scenarios/31-fstrim-enable.sh b/scripts/testing/maint-scenarios/31-fstrim-enable.sh new file mode 100644 index 0000000..2d548a1 --- /dev/null +++ b/scripts/testing/maint-scenarios/31-fstrim-enable.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: fstrim.timer enabled and started from the disabled default. +# +# The base image ships util-linux with fstrim.timer disabled — the exact +# state the metric flags. The remedy is systemctl enable --now. +SCENARIO_DESC="fstrim.timer enabled and started" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="btrfs" + +scenario_break() { + # Default state, but pin it so the scenario stays honest if a future + # base image enables the timer. + mexec "systemctl disable --now fstrim.timer >/dev/null 2>&1 || true; ! systemctl is-enabled fstrim.timer >/dev/null 2>&1" +} + +scenario_fix() { + mfix fstrim_enable +} + +scenario_assert() { + mexec "systemctl is-enabled fstrim.timer && systemctl is-active fstrim.timer" +} diff --git a/scripts/testing/maint-scenarios/32-cron-enable.sh b/scripts/testing/maint-scenarios/32-cron-enable.sh new file mode 100644 index 0000000..39d4de6 --- /dev/null +++ b/scripts/testing/maint-scenarios/32-cron-enable.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash disable=SC2034 # sourced by run-maint-scenarios.sh +# Scenario: cronie enabled and started (the stopped-cron posture). +# +# Bootstrap installs cronie but never enables it — the broken state is the +# package's default. The remedy enables + starts the service. +SCENARIO_DESC="cronie enabled and started from the stopped state" +SCENARIO_GROUP="systemd" +SCENARIO_PROFILES="any" + +scenario_break() { + mexec "systemctl disable --now cronie.service >/dev/null 2>&1 || true; ! systemctl is-active cronie.service >/dev/null 2>&1" +} + +scenario_fix() { + mfix cron_enable +} + +scenario_assert() { + mexec "systemctl is-enabled cronie.service && systemctl is-active cronie.service" +} diff --git a/scripts/testing/run-maint-nspawn.sh b/scripts/testing/run-maint-nspawn.sh new file mode 100644 index 0000000..04d4a32 --- /dev/null +++ b/scripts/testing/run-maint-nspawn.sh @@ -0,0 +1,268 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# Run pacman-level maint remedy scenarios in a systemd-nspawn container +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# The fast lane for the maint scenario suite: pure pacman-level cases (the +# "packages" scenario group — cache trims, orphan removal, pacnew deletion) +# don't need a booted machine, so they run in seconds against a pacstrap'd +# rootfs instead of minutes against the VM. Everything else (systemd, logs) +# needs pid 1 and journald — that stays in run-maint-scenarios.sh. +# +# The scenario files are shared with the VM runner and see the same helpers +# (mexec/mmaint/mfix/massert_metric); only the transport differs — the +# helper trio is deliberately duplicated between the two runners rather +# than abstracted over ssh-vs-nspawn. +# +# The rootfs is cached at $NSPAWN_ROOT and rebuilt with --fresh. Building +# and running need root (pacstrap, systemd-nspawn) via sudo. +# +# Usage: run-maint-nspawn.sh [--list] [--fresh] +# --list print the validated scenario plan and exit (no root needed) +# --fresh delete the cached rootfs and pacstrap a new one +# Env: MAINT_SCENARIO_DIR scenario dir (default: maint-scenarios/) +# MAINT_SRC maint package source tree +# (default: ~/.dotfiles/maint/src/maint) +# NSPAWN_ROOT rootfs dir (default: /var/tmp/archsetup-maint-nspawn) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" + +SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" +MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" +THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml" +NSPAWN_ROOT="${NSPAWN_ROOT:-/var/tmp/archsetup-maint-nspawn}" +NSPAWN_GROUP="packages" + +LIST_ONLY=false +FRESH=false + +usage() { + echo "Usage: $0 [--list] [--fresh]" + echo " --list print the validated scenario plan and exit" + echo " --fresh rebuild the cached rootfs from scratch" + echo "Env: MAINT_SCENARIO_DIR, MAINT_SRC, NSPAWN_ROOT" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --list) LIST_ONLY=true; shift ;; + --fresh) FRESH=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# ─── Plan: the pacman-level subset ─────────────────────────────────── +# Same contract validation as the VM runner; the selection rule is the +# group name — "packages" is pacman-level by construction. + +S_FILES=() S_NAMES=() S_DESCS=() + +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +build_plan() { + [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" + local f base + for f in "$SCENARIO_DIR"/*.sh; do + [ -e "$f" ] || break + _validate_scenario "$f" + [ "$(_scenario_var "$f" SCENARIO_GROUP)" = "$NSPAWN_GROUP" ] || continue + base=$(basename "$f" .sh) + S_FILES+=("$f") + S_NAMES+=("${base#[0-9][0-9]-}") + S_DESCS+=("$(_scenario_var "$f" SCENARIO_DESC)") + done + [ ${#S_FILES[@]} -gt 0 ] \ + || fatal "no '$NSPAWN_GROUP'-group scenarios in $SCENARIO_DIR" +} + +print_plan() { + local i + echo "maint nspawn scenario plan (group: $NSPAWN_GROUP)" + for i in "${!S_FILES[@]}"; do + echo " ${S_NAMES[$i]} — ${S_DESCS[$i]}" + done +} + +if [ "$LIST_ONLY" = "true" ]; then + LOGFILE=/dev/null + build_plan + print_plan + exit 0 +fi + +# ─── Run ───────────────────────────────────────────────────────────── + +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +RESULTS_DIR="$PROJECT_ROOT/test-results/maint-nspawn-$TIMESTAMP" +mkdir -p "$RESULTS_DIR" +LOGFILE="$RESULTS_DIR/scenarios.log" +init_logging "$LOGFILE" + +build_plan + +section "Maint Remedy Scenarios (nspawn): $TIMESTAMP" +info "Rootfs: $NSPAWN_ROOT" +info "Maint source: $MAINT_SRC" +print_plan | tee -a "$LOGFILE" + +command -v pacstrap >/dev/null \ + || fatal "pacstrap not found — pacman -S arch-install-scripts" +command -v systemd-nspawn >/dev/null || fatal "systemd-nspawn not found" +[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC" +[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML" +sudo -n true 2>/dev/null || fatal "needs passwordless sudo (pacstrap/nspawn)" + +# ─── Target helpers (available to scenario scripts) ────────────────── + +mexec() { # run a command in the container as root + sudo systemd-nspawn -q -D "$NSPAWN_ROOT" --pipe \ + /bin/bash -c "$*" 2>> "$LOGFILE" +} + +mmaint() { + mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*" +} + +mfix() { + local id="$1"; shift || true + step "maint fix $id ${*:+$* }--dry-run" + mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \ + || { error "dry-run failed: maint fix $id $*"; return 1; } + step "maint fix $id $*" + mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE" + return "${PIPESTATUS[0]}" +} + +massert_metric() { # <metric-id> <expected-severity> + local mid="$1" want="$2" got + got=$(mmaint status --json | python3 -c " +import json, sys +env = json.load(sys.stdin) +ms = [m for m in env['metrics'] if m['id'] == '$mid'] +print(ms[0]['severity'] if ms else 'MISSING') +") + if [ "$got" = "$want" ]; then + success "metric $mid is $want" + return 0 + fi + error "metric $mid: expected $want, got $got" + return 1 +} + +# ─── Rootfs ────────────────────────────────────────────────────────── + +if [ "$FRESH" = "true" ] && [ -d "$NSPAWN_ROOT" ]; then + step "Removing cached rootfs (--fresh)" + sudo rm -rf "$NSPAWN_ROOT" +fi + +if [ ! -x "$NSPAWN_ROOT/usr/bin/python3" ]; then + section "Building rootfs (pacstrap)" + sudo mkdir -p "$NSPAWN_ROOT" + # shellcheck disable=SC2024 # the log is user-owned by design + sudo pacstrap -c -K "$NSPAWN_ROOT" base python pacman-contrib expac \ + >> "$LOGFILE" 2>&1 || fatal "pacstrap failed (see $LOGFILE)" + success "Rootfs built" +else + info "Reusing cached rootfs (--fresh to rebuild)" +fi + +section "Installing maint tree + thresholds into the container" +sudo rm -rf "$NSPAWN_ROOT/root/maint-pkg" +sudo mkdir -p "$NSPAWN_ROOT/root/maint-pkg" "$NSPAWN_ROOT/root/.config/archsetup" +sudo cp -r "$MAINT_SRC" "$NSPAWN_ROOT/root/maint-pkg/" +sudo find "$NSPAWN_ROOT/root/maint-pkg" -name '__pycache__' -type d \ + -exec rm -rf {} + 2>/dev/null || true +sudo cp "$THRESHOLDS_TOML" \ + "$NSPAWN_ROOT/root/.config/archsetup/maintenance-thresholds.toml" + +step "Smoke: maint status --json runs in the container" +mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \ + || fatal "maint status failed in the container" +success "Container ready" + +# ─── Execution ─────────────────────────────────────────────────────── + +PASS=() FAIL=() + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} + +for i in "${!S_FILES[@]}"; do + run_scenario "$i" +done + +# ─── Report ────────────────────────────────────────────────────────── + +section "Results" +for n in "${PASS[@]}"; do success "PASS $n"; done +for n in "${FAIL[@]}"; do error "FAIL $n"; done +info "Log: $LOGFILE" +info "Rootfs kept for reuse: $NSPAWN_ROOT (--fresh to rebuild)" + +if [ ${#FAIL[@]} -gt 0 ]; then + error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed" + exit 1 +fi +success "All ${#PASS[@]} scenarios passed" +exit 0 diff --git a/scripts/testing/run-maint-scenarios.sh b/scripts/testing/run-maint-scenarios.sh new file mode 100644 index 0000000..448f6d9 --- /dev/null +++ b/scripts/testing/run-maint-scenarios.sh @@ -0,0 +1,374 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later +# Run maintenance-console remedy scenarios in the test VM +# Author: Craig Jennings <craigmartinjennings@gmail.com> +# License: GNU GPLv3 +# +# Layer 3 of the maint test strategy: each scenario breaks the VM in a known +# way over ssh, runs the real `maint fix <id>` inside the VM, and asserts the +# post-state. Scenarios live in maint-scenarios/*.sh and declare a GROUP; +# scenarios in one group are non-conflicting and share a VM boot, and a +# stop -> restore -> boot cycle runs only between groups (snapshot ops need +# the VM stopped). A scenario that fails may leave residue behind for its +# in-group successors -- the group restore boundary contains it, so read a +# multi-failure group front to back. +# +# Boot plan: +# 1. restore clean-install, boot, bootstrap (packages + maint tree + TOML) +# 2. stop, snapshot maint-ready +# 3. per group: boot, run scenarios, stop, restore maint-ready +# 4. restore clean-install, delete maint-ready (base stays pristine) +# +# Usage: run-maint-scenarios.sh [--list] [--group NAME] [--keep] +# --list print the validated batch plan and exit (no VM, no KVM needed) +# --group run (or list) only the named group +# --keep keep the VM running in its post-run state; skip the restores +# Env: FS_PROFILE=btrfs|zfs base image + scenario profile filter +# MAINT_SCENARIO_DIR scenario dir (default: maint-scenarios/) +# MAINT_SRC maint package source tree +# (default: ~/.dotfiles/maint/src/maint) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/vm-utils.sh" + +SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" +MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" +THRESHOLDS_TOML="$PROJECT_ROOT/configs/maintenance-thresholds.toml" +ROOT_PASSWORD="archsetup" +READY_SNAPSHOT="maint-ready" + +LIST_ONLY=false +ONLY_GROUP="" +KEEP_VM=false + +usage() { + echo "Usage: $0 [--list] [--group NAME] [--keep]" + echo " --list print the validated scenario plan and exit (no VM)" + echo " --group run only the named scenario group" + echo " --keep keep the VM running after the run (skip restores)" + echo "Env: FS_PROFILE=btrfs|zfs, MAINT_SCENARIO_DIR, MAINT_SRC" +} + +while [[ $# -gt 0 ]]; do + case $1 in + --list) LIST_ONLY=true; shift ;; + --group) ONLY_GROUP="${2:?--group requires a value}"; shift 2 ;; + --keep) KEEP_VM=true; shift ;; + *) usage; exit 1 ;; + esac +done + +# ─── Plan: enumerate, validate, group ──────────────────────────────── +# The plan layer is pure -- no VM, no KVM -- so --list works anywhere and +# the unit suite (tests/maint-scenarios/) exercises it directly. + +# Parallel arrays over scenarios, in file order. +S_FILES=() S_NAMES=() S_DESCS=() S_GROUPS=() +# Groups in first-appearance order. +GROUP_ORDER=() + +# Extract one declared var from a scenario file by sourcing it in a clean +# subshell. Scenario files must only define vars + functions; any top-level +# command fails here (no helpers exist), which is the contract. +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +_profile_matches() { # <profiles> -> 0 if scenario runs under $FS_PROFILE + local p + for p in $1; do + [ "$p" = "any" ] || [ "$p" = "$FS_PROFILE" ] && return 0 + done + return 1 +} + +build_plan() { + [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" + local f base name desc group profiles seen_groups=" " + local any=false + for f in "$SCENARIO_DIR"/*.sh; do + [ -e "$f" ] || break + any=true + _validate_scenario "$f" + base=$(basename "$f" .sh) + name="${base#[0-9][0-9]-}" + desc=$(_scenario_var "$f" SCENARIO_DESC) + group=$(_scenario_var "$f" SCENARIO_GROUP) + profiles=$(_scenario_var "$f" SCENARIO_PROFILES) + _profile_matches "$profiles" || continue + [ -n "$ONLY_GROUP" ] && [ "$group" != "$ONLY_GROUP" ] && continue + S_FILES+=("$f"); S_NAMES+=("$name") + S_DESCS+=("$desc"); S_GROUPS+=("$group") + case "$seen_groups" in + *" $group "*) ;; + *) GROUP_ORDER+=("$group"); seen_groups="$seen_groups$group " ;; + esac + done + [ "$any" = "true" ] || fatal "no scenario files in $SCENARIO_DIR" + if [ -n "$ONLY_GROUP" ] && [ ${#S_FILES[@]} -eq 0 ]; then + fatal "no scenarios match group '$ONLY_GROUP' (profile $FS_PROFILE)" + fi + [ ${#S_FILES[@]} -gt 0 ] \ + || fatal "no scenarios match profile $FS_PROFILE" +} + +print_plan() { + local g i + echo "maint scenario plan (profile: $FS_PROFILE)" + for g in "${GROUP_ORDER[@]}"; do + echo "group $g:" + for i in "${!S_FILES[@]}"; do + [ "${S_GROUPS[$i]}" = "$g" ] \ + && echo " ${S_NAMES[$i]} — ${S_DESCS[$i]}" + done + done +} + +# --list needs no logging file, VM, or KVM. Give fatal() a LOGFILE target +# only when we actually run. +if [ "$LIST_ONLY" = "true" ]; then + LOGFILE=/dev/null + build_plan + print_plan + exit 0 +fi + +# ─── Run ───────────────────────────────────────────────────────────── + +TIMESTAMP=$(date +'%Y%m%d-%H%M%S') +RESULTS_DIR="$PROJECT_ROOT/test-results/maint-$TIMESTAMP" +mkdir -p "$RESULTS_DIR" +LOGFILE="$RESULTS_DIR/scenarios.log" +init_logging "$LOGFILE" +init_vm_paths "$PROJECT_ROOT/vm-images" + +build_plan + +section "Maint Remedy Scenarios: $TIMESTAMP" +info "Profile: $FS_PROFILE (image: $(basename "$DISK_PATH"))" +info "Maint source: $MAINT_SRC" +print_plan | tee -a "$LOGFILE" + +[ -f "$DISK_PATH" ] || fatal "base disk not found: $DISK_PATH — build it: FS_PROFILE=$FS_PROFILE $SCRIPT_DIR/create-base-vm.sh" +snapshot_exists "$DISK_PATH" "clean-install" \ + || fatal "snapshot 'clean-install' not found on $DISK_PATH" +[ -d "$MAINT_SRC" ] || fatal "maint source tree not found: $MAINT_SRC" +[ -f "$THRESHOLDS_TOML" ] || fatal "thresholds TOML not found: $THRESHOLDS_TOML" +check_prerequisites || fatal "missing prerequisites" + +CLEANUP_DONE=0 +cleanup_scenarios() { + [ "$CLEANUP_DONE" = "1" ] && return 0 + CLEANUP_DONE=1 + [ "$KEEP_VM" = "true" ] && return 0 + # Never silent: a failed restore/delete here leaves the base image dirty + # for the next `make test`, so name it even though the trap can't abort. + stop_qemu 2>/dev/null \ + || echo "[!] cleanup: stop_qemu failed — VM may still hold the image" >&2 + restore_snapshot "$DISK_PATH" "clean-install" 2>/dev/null \ + || echo "[!] cleanup: clean-install restore FAILED — base image is dirty" >&2 + if snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT"; then + delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" 2>/dev/null \ + || echo "[!] cleanup: stray '$READY_SNAPSHOT' snapshot left on the base image" >&2 + fi +} +trap cleanup_scenarios EXIT + +# ─── Target helpers (available to scenario scripts) ────────────────── + +mexec() { # run a command in the VM as root + vm_exec "$ROOT_PASSWORD" "$@" +} + +# maint invocation inside the VM. Root runs the verbs directly, so +# MAINT_SUDO is empty (same switch the unit suites use). +mmaint() { + mexec "cd /root/maint-pkg && MAINT_SUDO= PYTHONPATH=/root/maint-pkg python3 -m maint $*" +} + +# Run one remedy: dry-run first (argv parity gate — the printed argv is the +# arm-press string), then for real. Fails if either exits non-zero. +mfix() { + local id="$1"; shift || true + step "maint fix $id ${*:+$* }--dry-run" + mmaint fix "$id" "$@" --dry-run >> "$LOGFILE" 2>&1 \ + || { error "dry-run failed: maint fix $id $*"; return 1; } + step "maint fix $id $*" + mmaint fix "$id" "$@" 2>&1 | tee -a "$LOGFILE" + return "${PIPESTATUS[0]}" +} + +# Assert a metric's severity in `maint status --json`. +massert_metric() { # <metric-id> <expected-severity> + local mid="$1" want="$2" got + got=$(mmaint status --json | python3 -c " +import json, sys +env = json.load(sys.stdin) +ms = [m for m in env['metrics'] if m['id'] == '$mid'] +print(ms[0]['severity'] if ms else 'MISSING') +") + if [ "$got" = "$want" ]; then + success "metric $mid is $want" + return 0 + fi + error "metric $mid: expected $want, got $got" + return 1 +} + +# ─── Bootstrap ─────────────────────────────────────────────────────── + +boot_vm() { + start_qemu "$DISK_PATH" "disk" "" "none" || fatal "failed to start VM" + wait_for_ssh "$ROOT_PASSWORD" 180 || fatal "VM SSH not available" +} + +bootstrap_vm() { + section "Bootstrap: packages + maint tree + thresholds" + step "Refreshing keyring + installing runtime deps" + # The base image can be months old: refresh the keyring first so the + # dep install doesn't die on expired signatures. + mexec "pacman -Sy --noconfirm archlinux-keyring" >> "$LOGFILE" 2>&1 \ + || fatal "keyring refresh failed (network up?)" + mexec "pacman -S --noconfirm python pacman-contrib expac cronie" \ + >> "$LOGFILE" 2>&1 || fatal "dep install failed" + + step "Copying maint package tree" + local tarball + tarball=$(mktemp) + tar -C "$(dirname "$MAINT_SRC")" --exclude='__pycache__' -czf "$tarball" \ + "$(basename "$MAINT_SRC")" + copy_to_vm "$tarball" "/tmp/maint-src.tgz" "$ROOT_PASSWORD" + rm -f "$tarball" + mexec "rm -rf /root/maint-pkg && mkdir -p /root/maint-pkg && tar -C /root/maint-pkg -xzf /tmp/maint-src.tgz && rm /tmp/maint-src.tgz" \ + >> "$LOGFILE" 2>&1 || fatal "maint tree extract failed" + + step "Installing thresholds TOML" + copy_to_vm "$THRESHOLDS_TOML" "/tmp/maintenance-thresholds.toml" \ + "$ROOT_PASSWORD" + mexec "mkdir -p /root/.config/archsetup && mv /tmp/maintenance-thresholds.toml /root/.config/archsetup/maintenance-thresholds.toml" \ + >> "$LOGFILE" 2>&1 || fatal "TOML install failed" + + step "Smoke: maint status --json runs" + mmaint status --json > "$RESULTS_DIR/bootstrap-status.json" \ + || fatal "maint status failed in the VM" + success "Bootstrap complete ($(python3 -c " +import json; print(len(json.load(open('$RESULTS_DIR/bootstrap-status.json'))['metrics']))") metrics probed)" +} + +# ─── Execution ─────────────────────────────────────────────────────── + +PASS=() FAIL=() + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # Sourced in the runner's shell so break/fix/assert see the helpers; + # unset afterwards so the next scenario can't inherit stale functions. + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} + +section "Preparing VM" +stop_qemu 2>/dev/null || true +step "Restoring clean-install snapshot" +restore_snapshot "$DISK_PATH" "clean-install" || fatal "restore failed" +boot_vm +bootstrap_vm + +step "Freezing bootstrap as snapshot: $READY_SNAPSHOT" +stop_qemu +snapshot_exists "$DISK_PATH" "$READY_SNAPSHOT" \ + && delete_snapshot "$DISK_PATH" "$READY_SNAPSHOT" +create_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "snapshot failed" + +first_group=true +for g in "${GROUP_ORDER[@]}"; do + section "Group: $g" + if [ "$first_group" = "true" ]; then + first_group=false + else + step "Isolation boundary: restore $READY_SNAPSHOT" + stop_qemu + restore_snapshot "$DISK_PATH" "$READY_SNAPSHOT" || fatal "restore failed" + fi + boot_vm + for i in "${!S_FILES[@]}"; do + [ "${S_GROUPS[$i]}" = "$g" ] && run_scenario "$i" + done + stop_qemu +done + +# ─── Report ────────────────────────────────────────────────────────── + +section "Results" +for n in "${PASS[@]}"; do success "PASS $n"; done +for n in "${FAIL[@]}"; do error "FAIL $n"; done +info "Log: $LOGFILE" + +if [ "$KEEP_VM" = "true" ]; then + warn "--keep: VM left in post-run state; base image NOT restored" + warn "restore by hand: qemu-img snapshot -a clean-install $DISK_PATH" + CLEANUP_DONE=1 +else + cleanup_scenarios +fi + +if [ ${#FAIL[@]} -gt 0 ]; then + error "${#FAIL[@]} scenario(s) failed, ${#PASS[@]} passed" + exit 1 +fi +success "All ${#PASS[@]} scenarios passed" +exit 0 diff --git a/tests/hypr-live-update-guard/test_hypr_live_update_guard.py b/tests/hypr-live-update-guard/test_hypr_live_update_guard.py index 5ec5ce8..a6c6f68 100644 --- a/tests/hypr-live-update-guard/test_hypr_live_update_guard.py +++ b/tests/hypr-live-update-guard/test_hypr_live_update_guard.py @@ -12,6 +12,10 @@ Test seams (env vars the production script honors): HYPR_GUARD_RUNNING 1/0 forces the Hyprland-running check (default: pgrep) HYPR_ALLOW_LIVE_UPDATE 1 overrides the guard (proceed anyway) HYPR_GUARD_SENTINEL path whose existence also overrides the guard + HYPR_GUARD_VERSIONS "pkg installed candidate" lines replacing the + pacman -Q / expac -S version lookups; when set, + a package absent from the map reads as unknown + (conservative block) Run from repo root: python3 -m unittest tests.hypr-live-update-guard.test_hypr_live_update_guard @@ -26,8 +30,18 @@ import unittest REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) GUARD = os.path.join(REPO_ROOT, "scripts", "hypr-live-update-guard") +# Every package the legacy tests feed, mapped to a version CHANGE — those +# tests describe real upgrades, and the map keeps them hermetic (no +# pacman/expac calls against the test host). +CHANGING_VERSIONS = "\n".join(( + "mesa 25.1.0-1 26.0.0-1", + "hyprland 0.55.3-1 0.55.4-1", + "vulkan-radeon 25.1.0-1 26.0.0-1", +)) -def run_guard(stdin="mesa\n", running="1", allow=None, sentinel=None): + +def run_guard(stdin="mesa\n", running="1", allow=None, sentinel=None, + versions=CHANGING_VERSIONS): env = dict(os.environ) env["HYPR_GUARD_RUNNING"] = running if allow is not None: @@ -35,6 +49,7 @@ def run_guard(stdin="mesa\n", running="1", allow=None, sentinel=None): # Point the sentinel at a path that does not exist unless a test sets one, # so the host's real /run state can't leak into the result. env["HYPR_GUARD_SENTINEL"] = sentinel if sentinel else "/nonexistent/guard-sentinel" + env["HYPR_GUARD_VERSIONS"] = versions return subprocess.run( ["sh", GUARD], input=stdin, capture_output=True, text=True, timeout=10, env=env, @@ -75,6 +90,42 @@ class HyprLiveUpdateGuard(unittest.TestCase): r = run_guard(stdin="", running="1") self.assertEqual(r.returncode, 1) + # --- Version awareness ------------------------------------------------ + + def test_same_version_reinstall_allows(self): + # a pure reinstall replaces identical bytes with identical bytes — + # no live-swap hazard, the guard must let it through + r = run_guard(stdin="hyprland\n", running="1", + versions="hyprland 0.55.4-1 0.55.4-1") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stderr.strip(), "") + + def test_all_same_version_multi_pkg_allows(self): + versions = "\n".join(("mesa 26.1.4-1 26.1.4-1", + "hyprland 0.55.4-1 0.55.4-1", + "vulkan-radeon 26.1.4-1 26.1.4-1")) + r = run_guard(stdin="mesa\nhyprland\nvulkan-radeon\n", running="1", + versions=versions) + self.assertEqual(r.returncode, 0, r.stderr) + + def test_mixed_blocks_naming_only_the_version_changing(self): + versions = "\n".join(("mesa 25.1.0-1 26.0.0-1", + "hyprland 0.55.4-1 0.55.4-1")) + r = run_guard(stdin="mesa\nhyprland\n", running="1", + versions=versions) + self.assertEqual(r.returncode, 1) + self.assertIn("- mesa", r.stderr) + # the prose mentions the Hyprland session, so assert on the + # package-list line format, not the bare word + self.assertNotIn("- hyprland", r.stderr) + + def test_unknown_versions_block_conservatively(self): + # seam set but package absent from the map = the lookup failed + # (AUR target, -U transaction, expac missing) — stay safe + r = run_guard(stdin="mesa\n", running="1", versions="") + self.assertEqual(r.returncode, 1) + self.assertIn("- mesa", r.stderr) + # --- Override / error cases ----------------------------------------- def test_env_override_proceeds_even_when_running(self): @@ -86,6 +137,25 @@ class HyprLiveUpdateGuard(unittest.TestCase): r = run_guard(stdin="mesa\n", running="1", sentinel=f.name) self.assertEqual(r.returncode, 0, r.stderr) + def test_sentinel_is_consumed_on_use(self): + # one touch = one transaction: if the override's cleanup never runs + # (a crashed caller), a leftover sentinel must not keep the guard + # disarmed until reboot — the hook deletes it as it honors it + fd, path = tempfile.mkstemp(prefix="guard-allow-") + os.close(fd) + try: + r = run_guard(stdin="mesa\n", running="1", sentinel=path) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertFalse(os.path.exists(path)) + finally: + if os.path.exists(path): + os.unlink(path) + + def test_env_override_consumes_nothing(self): + # the env override isn't a file; nothing to consume, still proceeds + r = run_guard(stdin="mesa\n", running="1", allow="1") + self.assertEqual(r.returncode, 0, r.stderr) + def test_override_env_zero_does_not_bypass(self): r = run_guard(stdin="mesa\n", running="1", allow="0") self.assertEqual(r.returncode, 1, r.stderr) diff --git a/tests/installer-steps/test_orchestrators.py b/tests/installer-steps/test_orchestrators.py index 48b7508..2a771ba 100644 --- a/tests/installer-steps/test_orchestrators.py +++ b/tests/installer-steps/test_orchestrators.py @@ -51,7 +51,8 @@ ORCHESTRATORS = { "user_customizations": [ "clone_user_repos", "stow_dotfiles", "prune_waybar_battery", "refresh_desktop_caches", "configure_dconf_defaults", - "finalize_dotfiles", "create_user_directories", + "finalize_dotfiles", "install_maintenance_config", + "create_user_directories", ], } @@ -114,5 +115,35 @@ class SnapshotDispatch(unittest.TestCase): self.assertEqual(result.stdout.split(), []) +class MaintenanceConfigDispatch(unittest.TestCase): + """install_maintenance_config branches on desktop_env; pin each branch. + + The thresholds TOML installs for every environment (the CLI works + headless); the scan timers are user units in the hyprland stow tier, so + their enablement is hyprland-only. + """ + + SUBS = ["install_maintenance_thresholds", "enable_maint_timers"] + + def test_hyprland_installs_thresholds_and_enables_timers(self): + result = run_orchestrator( + "install_maintenance_config", self.SUBS, + extra_defs='desktop_env=hyprland', + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.split(), self.SUBS) + + def test_non_hyprland_installs_thresholds_only(self): + for env in ("dwm", "none"): + with self.subTest(desktop_env=env): + result = run_orchestrator( + "install_maintenance_config", self.SUBS, + extra_defs=f'desktop_env={env}', + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.split(), + ["install_maintenance_thresholds"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/installer-steps/test_pacman_install.py b/tests/installer-steps/test_pacman_install.py new file mode 100644 index 0000000..28c9a7f --- /dev/null +++ b/tests/installer-steps/test_pacman_install.py @@ -0,0 +1,95 @@ +"""Characterization tests for pacman_install's install-reason handling. + +pacman --needed skips a package that is already present as a dependency and +leaves its install reason alone. A declared package can then sit as asdeps +on an existing system, show up as an orphan once its accidental dependent +leaves, and get swept away by an orphan cleanup (expac and lm_sensors nearly +went this way on 2026-07-08). pacman_install therefore marks every declared +package explicit after a successful install. + +Method mirrors test_orchestrators: sed-extract the real functions from +`archsetup`, source them with `display`/`error_warn` silenced and `pacman` +replaced by a recorder, run, and assert the recorded calls. + +Run from repo root: + python3 -m unittest tests.installer-steps.test_pacman_install +""" + +import os +import subprocess +import textwrap +import unittest + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +ARCHSETUP = os.path.join(REPO_ROOT, "archsetup") + + +def run_pacman_install(pkg, pacman_s_rc=0, pacman_d_rc=0): + """Extract retry_install + pacman_install, run against a fake pacman. + + Returns (exit_code, recorded pacman calls as a list of strings). + """ + script = textwrap.dedent(""" + set -u + MAX_INSTALL_RETRIES=3 + logfile=/dev/null + display() { :; } + error_warn() { return 1; } + pacman() { + echo "pacman $*" >> "$CALLS" + case "$1" in + --noconfirm) return "$PACMAN_S_RC" ;; + -D) return "$PACMAN_D_RC" ;; + esac + } + %(functions)s + pacman_install "%(pkg)s" + """) + extract = subprocess.run( + ["sed", "-n", + "/^retry_install()/,/^}/p;/^pacman_install()/,/^}/p", ARCHSETUP], + capture_output=True, text=True, check=True) + calls_file = os.path.join(os.environ.get("TMPDIR", "/tmp"), + f"pacman-install-calls-{os.getpid()}") + if os.path.exists(calls_file): + os.unlink(calls_file) + open(calls_file, "w").close() + env = dict(os.environ, CALLS=calls_file, + PACMAN_S_RC=str(pacman_s_rc), PACMAN_D_RC=str(pacman_d_rc)) + proc = subprocess.run( + ["bash", "-c", script % {"functions": extract.stdout, "pkg": pkg}], + capture_output=True, text=True, env=env) + with open(calls_file) as f: + calls = [line.strip() for line in f if line.strip()] + os.unlink(calls_file) + return proc.returncode, calls + + +class PacmanInstallTests(unittest.TestCase): + def test_success_marks_package_explicit(self): + rc, calls = run_pacman_install("expac") + self.assertEqual(rc, 0) + self.assertIn("pacman --noconfirm --needed -S expac", calls) + self.assertIn("pacman -D --asexplicit expac", calls) + # the mark comes after the install, never before + self.assertGreater(calls.index("pacman -D --asexplicit expac"), + calls.index("pacman --noconfirm --needed -S expac")) + + def test_failed_install_never_marks(self): + rc, calls = run_pacman_install("expac", pacman_s_rc=1) + self.assertNotEqual(rc, 0) + self.assertNotIn("pacman -D --asexplicit expac", calls) + # all three retry attempts happened + self.assertEqual( + calls.count("pacman --noconfirm --needed -S expac"), 3) + + def test_mark_failure_does_not_fail_the_install(self): + # -D can fail in odd corners (readonly db mid-transaction); the + # install itself succeeded and must report success + rc, calls = run_pacman_install("expac", pacman_d_rc=1) + self.assertEqual(rc, 0) + self.assertIn("pacman -D --asexplicit expac", calls) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/maint-scenarios/test_scenario_plan.py b/tests/maint-scenarios/test_scenario_plan.py new file mode 100644 index 0000000..9a72db2 --- /dev/null +++ b/tests/maint-scenarios/test_scenario_plan.py @@ -0,0 +1,273 @@ +"""Tests for the maint VM scenario runner's plan layer (no VM needed). + +run-maint-scenarios.sh orchestrates break -> `maint fix` -> assert scenario +scripts over the existing qemu-img snapshot primitives (lib/vm-utils.sh). +Scenarios are grouped into non-conflicting batches that share one VM boot; +a stop -> restore -> boot cycle runs only between groups (the spec's +grouped-batch isolation policy). The runner therefore has a pure planning +layer -- enumerate scenario files, validate their contract, filter by +filesystem profile and --group, and print the batch plan -- that runs +without KVM, a base image, or root. + +These tests exercise that layer through the REAL script via `--list`: + - against the shipped scenarios directory (contract holds for every file + we actually ship); + - against fake scenario directories (MAINT_SCENARIO_DIR override) for the + validation failures a shipped tree must never contain. + +The scenario-file contract validated here: + - vars SCENARIO_DESC (non-empty), SCENARIO_GROUP (token), + SCENARIO_PROFILES (btrfs/zfs/any, space-separated); + - functions scenario_break, scenario_fix, scenario_assert; + - defining only -- sourcing a scenario file must not execute commands + (the probe sources files in a bare shell with no helpers defined). + +Run from repo root: + python3 -m unittest tests.maint-scenarios.test_scenario_plan +""" + +import os +import subprocess +import tempfile +import unittest + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +RUNNER = os.path.join(REPO_ROOT, "scripts", "testing", "run-maint-scenarios.sh") +SCENARIO_DIR = os.path.join(REPO_ROOT, "scripts", "testing", "maint-scenarios") + +GOOD_SCENARIO = """\ +SCENARIO_DESC="{desc}" +SCENARIO_GROUP="{group}" +SCENARIO_PROFILES="{profiles}" +scenario_break() {{ mexec "true"; }} +scenario_fix() {{ mfix some_remedy; }} +scenario_assert() {{ mexec "true"; }} +""" + + +def run_list(extra_args=(), scenario_dir=None, fs_profile=None): + # Hermetic against the caller's FS_PROFILE: the Makefile exports it, so + # `make test-unit FS_PROFILE=zfs` would otherwise change what --list + # shows. Tests that care pass fs_profile explicitly; everything else + # runs the runner's own default (btrfs). + env = {k: v for k, v in os.environ.items() if k != "FS_PROFILE"} + if scenario_dir is not None: + env["MAINT_SCENARIO_DIR"] = scenario_dir + if fs_profile is not None: + env["FS_PROFILE"] = fs_profile + return subprocess.run( + ["bash", RUNNER, "--list", *extra_args], + capture_output=True, text=True, env=env, cwd=REPO_ROOT, + ) + + +def write_scenario(dirpath, name, desc="a scenario", group="g1", + profiles="any", body=None): + path = os.path.join(dirpath, name) + with open(path, "w") as f: + f.write(body if body is not None + else GOOD_SCENARIO.format(desc=desc, group=group, + profiles=profiles)) + return path + + +class ShippedScenariosTests(unittest.TestCase): + """The scenarios we actually ship satisfy the contract.""" + + def test_shipped_dir_exists_and_is_nonempty(self): + files = [f for f in os.listdir(SCENARIO_DIR) if f.endswith(".sh")] + self.assertTrue(files, "no scenario files shipped") + + def test_list_exits_zero_on_shipped_scenarios(self): + proc = run_list() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_list_names_every_shipped_scenario(self): + proc = run_list() + for f in os.listdir(SCENARIO_DIR): + if not f.endswith(".sh"): + continue + name = f.split("-", 1)[1][:-3] if "-" in f else f[:-3] + self.assertIn(name, proc.stdout, + f"scenario {f} missing from --list output") + + def test_list_groups_are_headed(self): + proc = run_list() + self.assertRegex(proc.stdout, r"(?m)^group \S+:") + + def test_shipped_scenarios_define_contract_without_executing(self): + """Sourcing a scenario file in a bare bash defines the contract vars + and functions and runs nothing (no helpers exist at source time, so + any top-level command would fail loudly).""" + probe = ( + 'set -eu; source "$1"; ' + ': "${SCENARIO_DESC:?}" "${SCENARIO_GROUP:?}" ' + '"${SCENARIO_PROFILES:?}"; ' + 'case " $SCENARIO_PROFILES " in *" btrfs "*|*" zfs "*|*" any "*) ' + ';; *) echo "bad profiles: $SCENARIO_PROFILES" >&2; exit 1;; esac; ' + 'declare -f scenario_break scenario_fix scenario_assert >/dev/null' + ) + for f in sorted(os.listdir(SCENARIO_DIR)): + if not f.endswith(".sh"): + continue + path = os.path.join(SCENARIO_DIR, f) + proc = subprocess.run(["bash", "-c", probe, "probe", path], + capture_output=True, text=True) + self.assertEqual(proc.returncode, 0, + f"{f}: contract violation\n{proc.stderr}") + + +class PlanFilteringTests(unittest.TestCase): + """Profile and --group filtering over a fake scenario dir.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = self.tmp.name + write_scenario(self.dir, "10-first-btrfs.sh", + group="alpha", profiles="btrfs") + write_scenario(self.dir, "20-second-any.sh", + group="beta", profiles="any") + write_scenario(self.dir, "30-third-zfs.sh", + group="gamma", profiles="zfs") + + def tearDown(self): + self.tmp.cleanup() + + def test_btrfs_profile_excludes_zfs_scenarios(self): + proc = run_list(scenario_dir=self.dir, fs_profile="btrfs") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("first-btrfs", proc.stdout) + self.assertIn("second-any", proc.stdout) + self.assertNotIn("third-zfs", proc.stdout) + + def test_zfs_profile_excludes_btrfs_scenarios(self): + proc = run_list(scenario_dir=self.dir, fs_profile="zfs") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertNotIn("first-btrfs", proc.stdout) + self.assertIn("second-any", proc.stdout) + self.assertIn("third-zfs", proc.stdout) + + def test_group_filter_selects_one_group(self): + proc = run_list(["--group", "alpha"], + scenario_dir=self.dir, fs_profile="btrfs") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("first-btrfs", proc.stdout) + self.assertNotIn("second-any", proc.stdout) + + def test_unknown_group_is_an_error(self): + proc = run_list(["--group", "nonesuch"], + scenario_dir=self.dir, fs_profile="btrfs") + self.assertNotEqual(proc.returncode, 0) + self.assertIn("nonesuch", proc.stdout + proc.stderr) + + def test_groups_appear_in_file_order(self): + proc = run_list(scenario_dir=self.dir, fs_profile="zfs") + out = proc.stdout + self.assertLess(out.index("group beta:"), out.index("group gamma:")) + + +class ContractValidationTests(unittest.TestCase): + """Malformed scenario files fail the plan, naming the file.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = self.tmp.name + + def tearDown(self): + self.tmp.cleanup() + + def assert_plan_fails_naming(self, filename): + proc = run_list(scenario_dir=self.dir) + self.assertNotEqual(proc.returncode, 0, + f"plan accepted malformed {filename}") + self.assertIn(filename, proc.stdout + proc.stderr) + + def test_missing_desc_rejected(self): + write_scenario(self.dir, "10-no-desc.sh", body=( + 'SCENARIO_GROUP="g"\nSCENARIO_PROFILES="any"\n' + "scenario_break() { :; }\nscenario_fix() { :; }\n" + "scenario_assert() { :; }\n")) + self.assert_plan_fails_naming("10-no-desc.sh") + + def test_missing_function_rejected(self): + write_scenario(self.dir, "10-no-assert.sh", body=( + 'SCENARIO_DESC="d"\nSCENARIO_GROUP="g"\nSCENARIO_PROFILES="any"\n' + "scenario_break() { :; }\nscenario_fix() { :; }\n")) + self.assert_plan_fails_naming("10-no-assert.sh") + + def test_bad_profile_token_rejected(self): + write_scenario(self.dir, "10-bad-profile.sh", profiles="ext4") + self.assert_plan_fails_naming("10-bad-profile.sh") + + def test_empty_scenario_dir_is_an_error(self): + proc = run_list(scenario_dir=self.dir) + self.assertNotEqual(proc.returncode, 0) + + +class UsageTests(unittest.TestCase): + def test_unknown_flag_is_an_error_with_usage(self): + proc = run_list(["--bogus"], scenario_dir=SCENARIO_DIR) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Usage", proc.stdout + proc.stderr) + + +NSPAWN_RUNNER = os.path.join(REPO_ROOT, "scripts", "testing", + "run-maint-nspawn.sh") + + +class NspawnPlanTests(unittest.TestCase): + """The nspawn fast lane selects exactly the pacman-level (packages) + group from the shared scenario dir.""" + + def run_nspawn_list(self, scenario_dir=None): + env = dict(os.environ) + if scenario_dir is not None: + env["MAINT_SCENARIO_DIR"] = scenario_dir + return subprocess.run( + ["bash", NSPAWN_RUNNER, "--list"], + capture_output=True, text=True, env=env, cwd=REPO_ROOT, + ) + + def test_list_exits_zero(self): + proc = self.run_nspawn_list() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_list_selects_only_the_packages_group(self): + proc = self.run_nspawn_list() + listed = set() + for f in os.listdir(SCENARIO_DIR): + if not f.endswith(".sh"): + continue + name = f.split("-", 1)[1][:-3] if "-" in f else f[:-3] + group = subprocess.run( + ["bash", "-c", f'source "{os.path.join(SCENARIO_DIR, f)}"; ' + 'printf %s "$SCENARIO_GROUP"'], + capture_output=True, text=True).stdout + if group == "packages": + self.assertIn(name, proc.stdout, + f"packages scenario {f} missing") + listed.add(name) + else: + self.assertNotIn(name, proc.stdout, + f"non-packages scenario {f} listed") + self.assertTrue(listed, "no packages-group scenarios found") + + def test_unknown_flag_is_an_error_with_usage(self): + proc = subprocess.run(["bash", NSPAWN_RUNNER, "--bogus"], + capture_output=True, text=True, cwd=REPO_ROOT) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("Usage", proc.stdout + proc.stderr) + + def test_bad_profile_token_rejected_like_the_vm_lane(self): + """Both runners enforce the same scenario contract — a profile typo + must not pass the nspawn plan and only surface in the VM lane.""" + with tempfile.TemporaryDirectory() as d: + write_scenario(d, "10-bad-profile.sh", group="packages", + profiles="ext4") + proc = self.run_nspawn_list(scenario_dir=d) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("10-bad-profile.sh", proc.stdout + proc.stderr) + + +if __name__ == "__main__": + unittest.main() @@ -21,7 +21,8 @@ The vocabulary is open — topic tags are coined as needed — so these are conv - *Effort / autonomy*: =:quick:= a spare-moment fix (minutes, not a sitting); =:solo:= Claude can carry it end to end — there's a build path, a test path, and no upfront decision needed (a leftover manual spot-check doesn't disqualify it). - *Topic / area* (open): the subsystem a task touches — e.g. =:hyprland:= =:waybar:= =:mpd:= =:music:= =:network:= =:tooling:= =:llm:= =:eask:= =:pocketbook:= =:cmail:=. Coin a new one when it aids filtering. * Archsetup Open Work -** DOING [#B] Maintenance console build :feature: +** DONE [#B] Maintenance console build :feature: +CLOSED: [2026-07-08 Wed] :PROPERTIES: :SPEC_ID: 9d9df833-c592-4aec-a7df-50d588e943ce :END: @@ -59,28 +60,49 @@ Failed-unit roster (armed RESTART/RESET per unit, since/exit context, journalctl *** 2026-07-08 Wed @ 02:36:54 -0500 Built Phase 10 — results wall + doctor keys (dotfiles cc2fb5d) Results wall shipped as the session action log: doctor events stream per-event (begin opens a running entry with the exact argv, ok/fail resolves in place, re-probe notes follow), 3.5-visible-row scroll cap with autoscroll, HIDE/COPY (COPY emits CLI-wall text), curation writes logged (fixture-suppressed as dim notes). CLEAN UP streams the auto tier with no arm press; REVIEW & FIX swaps the subpanel to the confirm-tier roster (item-less remedies get armed FIX keys, per-item ones point at their subpanel; review events carry item + worst severity). RECLAIM SPACE macro landed on the disk-usage row (deferred from 9b); macro arm line shows first step + remaining count; zero-step macros refuse by name. Review (live-verified): Request Changes — 1 Important: the roster's FIX for update/topgrade could never override the live-update guard (fire-time re-arm used the bare rid while the FIX key arms on "rid:", and _on_lever never passed force — an infinite arm→guard loop); fixed by threading the pressed token through _fire and sharing the strip's guard arming (_guard_arm_line + panel.guarded). Minors all taken: wall_abort resolves stranded running entries red when a stream dies, empty-macro IndexError → named RemedyError, test predicate tightened. Re-review: Approve, no residuals. maint 612→645; make test 59 suites exit 0 ×2; AT-SPI smoke extended (wall, HIDE/SHOW, CLEAN UP stream, review roster, 14-line RECLAIM stream) PASS; screenshots vs E5; LIVE verify on ratio: real CLEAN UP on the live board streamed "3 done" with real re-probes (coredumps warn 5, disk_usage ok 60). Velox pulled cc2fb5d. -*** TODO Phase 11 — waybar glyph + timers :solo: -indicator.py + waybar-maint, custom/maint replaces custom/sysmon (canonical + waybar-active-config + SIGUSR2), right-click re-homes pypr toggle monitor, battery text on type==Battery hosts (low-charge threshold feeds diagnostic), maint-scan.timer + maint-net-scan.timer, glyph CSS both themes. Retire waybar-sysmon + sysmon-cycle + their suites + #custom-sysmon CSS (3 files) same commit. Verify: glyph live on ratio through a timer cycle; battery text + one-time stow on velox over tailscale. +*** 2026-07-08 Wed @ 03:24:44 -0500 Built Phase 11 — waybar glyph + timers (dotfiles 10033be) +custom/maint replaced custom/sysmon: waybar-maint renders from the state file maint-scan.timer writes every 30 min; color tracks the worst diagnostic (lever-less) metric only, actionable findings never color the bar; missing/stale scan data dims to an honest unprobed state naming the timer. Battery hosts show live battery % + charging icon from sysfs on waybar's interval; low charge forces red. Left-click toggles the console (new maint-panel wrapper); right-click keeps the btop scratchpad. Bare =maint scan= became the glyph scan (envelope → state → RTMIN+12); maint-net-scan.timer drives =--net --slow= hourly. Retired same commit: waybar-sysmon, sysmon-cycle, both suites, #custom-sysmon CSS ×3 — plus unnamed collateral the grep caught: waybar-collapse's collapsed base set (maint glyph stays on battery hosts — it IS the battery display now) and a stale dotfiles todo. Review (live-verified): Approve with 4 Minor, all taken red-first — scope!=Device gate so a wireless mouse's battery can't masquerade as the machine's (shared capability.device_scope), unprobed diagnostics named in the tooltip instead of "all clear" under a dimmed glyph, comment truth, dead state field dropped; re-confirm clean (677 maint tests). Full make test exit 0; smoke PASS (first run's pipe masked a transient AT-SPI race — rerun unpiped clean). LIVE both hosts: ratio timer fired for real (glyph amber from genuine diagnostics: firmware update + journal errors; 11 actionable NOT coloring), velox pulled + one-time stow + battery 100% text w/ warning class. velox's user systemd manager found wedged (pre-existing) — timers enabled via manual symlinks + state seeded; Craig rebooting to clear. -*** TODO Phase 12 — VM remedy scenarios + AT-SPI target :solo: -Scenario orchestration over vm-utils.sh snapshot primitives (grouped batches per Decision 11), scenario scripts (break → =maint fix= → assert), nspawn variant for pacman-level, FS_PROFILE=zfs scenario if the base image builds, test-panel-maint Makefile target. May split VM (archsetup) from AT-SPI (dotfiles). Verify: full scenario run green from pristine snapshot. +*** 2026-07-08 Wed @ 04:15:45 -0500 Built Phase 11b — prototype fidelity pass (dotfiles 3ee22a8) +Rebuilt the GUI presentation to the E5 instrument-card idiom. Each band renders a 4-column card grid: big-number readouts (viewmodel.card_spec registry — one builder per bespoke metric id, count-card fallback for the rest), cairo progress bars w/ threshold ticks (cache scaled to 1.5× the warn line), radial gauges (scrub cadence, keyring/topgrade age, temperatures — sweep scaled to the metric's own threshold), status chips (PASSED/FAILING/INACTIVE/YES; firewall chips the state word, exposure detail moves to the caption), and the CPU-mode segmented control promoted from digest section to card (current mode lit, alternatives as joined armed keys — same DIGEST_ONLY epp_set arm-then-fire). Evidence digests stay full-width below the grid. Selector became a two-row 4-col grid of wide tiles w/ count chips (3! · 7✓, crit reddens) + severity left borders; subpanel header carries the attention/ok/fixable/watch summary right-aligned beside a gold band title. The scoped NVMe wear dial was skipped honestly — no standalone wear metric exists (wear rides SMART evidence rows). Presentation only: engine/remedies/doctor/curation and the lever contracts unchanged. New 42-test suite (card kinds, threshold scales, running-scrub override, unprobed degrade incl. tick drop, seg exclusions, chips/subhead/band_counts, both-fixture sweep). Review (live-verified): Approve w/ 1 Important (pre-existing: chrony "not running" string crashed format_value's ms branch — fixed w/ card guard) + 4 Minor all taken red-first (disk None guard, _pct negative clamp, unprobed tick, dead row_detail/split_label/.maint-val removed); re-review Approve. make test 61 suites exit 0 ×2 unpiped; AT-SPI smoke 92 checks PASS (new gauge/chip/subhead assertions — DrawingAreas carry accessible_role=IMG + names so the smoke can see them); all 8 bands + review roster screenshotted on both fixtures + the live board against the settled headless-Chrome prototype render. Velox pulled (no restow). Gotcha pinned: a card body's vexpand propagates up a Gtk.Grid — a one-card band ballooned until grid.set_vexpand(False). -*** TODO Phase 13 — install wiring, workflow move, docs :solo: -archsetup installs the TOML (idempotent, never overwrites user layer); deps; maint README (user + developer); move system-health-check.org + homelab-inventory into archsetup, rewire to the TOML, handoff note to home's inbox for the home-side removal. Verify: fresh-shell =maint status= works from installed paths; docs build/read. +*** 2026-07-08 Wed @ 05:05:20 -0500 Ran the granted /refactor sweep over the maint package (dotfiles a178470..73e9d94) +Four parallel read-only scans (complexity, duplication, dead-code, simplification) converged on the same hotspots; ten behavior-preserving commits applied and pushed. Headlines: six curation handlers folded into one _curation helper (~150 lines of repeated fixture/done plumbing, act/wall strings byte-identical), _on_lever/_on_strip_key merged into _press_lever, _on_fired split (_rearm_after_guard + _fire_summary), the "tool: not found / exit N" wording single-sited in cmd.why() across 20 probe sites, exact clones deduped (packages curation_entries, _parse_asctime, double VERSION), ten uniform digest sections folded into _evidence_digest, _card_body split per kind, listener key rule named, plus a small-simplifications batch. Deliberately skipped: format_value's flat chain (readable as-is), _journal_lines generalization (rc-semantics risk), test-panel-maint Makefile target (Phase 12 scope). Verification: suite parity exact pre/post (59 suites / 2490 tests, exit 0 unpiped), AT-SPI smoke PASS end-to-end, live ratio envelope 45 metrics zero unprobed, review subagent walked the full diff against 3ee22a8 and approved (no behavior change; accessible names untouched; its one latent-robustness minor pinned as an invariant comment). gui.py 1680→1605; net −103 lines. Velox pulled. -*** TODO Manual testing and validation -Collects everything not agent-verifiable; populate per verification.md as phases land. Known so far: panel look-and-feel vs the E5 prototype (Craig's eyeball); arm-press wording reads right at the moment of use; results-wall readability during a real doctor run; a real UPDATE through the armed guard (and the TTY path once); REBOOT offer after an update; velox in-person glyph check (battery %, charging glyph, low-charge red); SET 80% charge limit sticks across a charge cycle; KILL on a real memory hog. +*** 2026-07-08 Wed @ 05:46:02 -0500 Built Phase 12 — VM remedy scenarios + AT-SPI target (archsetup d6993d3, dotfiles 0636554) +Scenario orchestration landed as =run-maint-scenarios.sh= over the existing vm-utils.sh snapshot primitives: nine break → =maint fix= → assert scenarios in three groups (logs, packages, systemd), batched per Decision 11 (one boot per group, maint-ready snapshot restored between groups, clean-install restored at the end). =run-maint-nspawn.sh= is the fast lane — the packages group against a cached pacstrap rootfs in seconds. Plan layer is pure (=--list=, no KVM) with a 19-test unit suite; =make test-maint= wires it in. Dotfiles side: panel smoke now runs GOOD + BAD fixture passes (healthy render, badge tallies from the fixture, no-REBOOT, leak check) and =make test-panel-maint= is enumerated. Verified: full VM run 9/9 green from pristine snapshot ×2 (test-results/maint-20260708-054029), nspawn 4/4, smoke 100 checks exit 0, unit suite green under both FS_PROFILE values. Review subagent: Request Changes (FS_PROFILE env leak in the unit suite, journal-vacuum assert vacuous both directions) — all six findings fixed red-first, re-review Approve. FS_PROFILE=zfs scenarios: filtered but unexercised — the zfs base image fails to build (see the zfs DKMS task). + +*** 2026-07-08 Wed @ 06:18:58 -0500 Built Phase 13 — install wiring, workflow move, docs (archsetup bef7053 + 18c081f, dotfiles 9a3f0c7) +Installer: install_maintenance_config in user_customizations installs the shipped TOML to ~/.config/archsetup/ (re-run refreshes it, ~/.config/maint/ user layer never touched) and enables maint-scan/maint-net-scan timers via wants-symlinks (hyprland-only — the units live in that stow tier; no session bus during install, syncthing idiom); orchestrator + dispatch-branch tests pin the wiring. Deps swept from the probes' argv: expac, lm_sensors, fwupd added (arch-audit was already in from Phase 3; the rest ride base/archinstall or existing sections). maint/README.md in dotfiles covers user (CLI, panel, glyph, config layers, guard, privilege) + developer (module map, env seams, four test layers, add-a-metric recipe). system-health-check.org moved home → archsetup docs/workflows/ and rewired: TOML-authoritative severity section, threshold lines cite TOML keys (TOML wins on disagreement), inventory paths → docs/homelab-inventory/, maint-CLI fast-path note; only the four #+HOSTNAME: host inventories moved (+ the truenas specs asset they link) — the personal gear records stay in home per the domain split; discoverability via .ai/project-workflows symlink + notes.org entry; handoff note in home's inbox (2026-07-08-0607) lists home-side removals and flags strix-soak-watch as a candidate to move later. Verified: fresh-shell =maint status= green from installed paths on ratio; make test-unit exit 0; dotfiles make test exit 0; shellcheck count unchanged; review subagent Approve (2 Minor doc nits, both taken). + +*** 2026-07-08 Wed @ 06:18:58 -0500 Flipped the spec to IMPLEMENTED +Spec keyword DOING → IMPLEMENTED with a dated history line naming the closing commits (phases 1-13, dotfiles 43a39ac..9a3f0c7, archsetup d6993d3/bef7053/18c081f) and the Status mirror updated. Residuals tracked as their own tasks: manual-testing checklist, zfs base-image DKMS bug [#C], vNext [#D]. + +** TODO [#B] Manual testing and validation: maintenance console :test: +Promoted from the build parent when it closed 2026-07-08 — Craig's checklist, runs once in person. Collects everything not agent-verifiable; populate per verification.md as phases land. Known so far: panel look-and-feel vs the E5 prototype (Craig's eyeball); arm-press wording reads right at the moment of use; results-wall readability during a real doctor run; a real UPDATE through the armed guard (and the TTY path once); REBOOT offer after an update; velox in-person glyph check (battery %, charging glyph, low-charge red); SET 80% charge limit sticks across a charge cycle; KILL on a real memory hog. Phase 8 additions: subpanel scroll position survives an arm press on a long digest (MAINT_PANEL_FIXTURE=bad, PACKAGES band, scroll into the 51-orphan list, press REMOVE — the list should stay put and the key read REMOVE?); a KEEP on a real orphan dims it into the kept section and UNKEEP restores it; MERGE on a needs-merge pacnew opens pacdiff in a foot terminal. Phase 9a additions: MARK KNOWN on a real journal group (ratio's wpa_supplicant bgscan spam is the standing candidate) — arm shows the identifier + sample wording, fire writes curation.toml, the next re-scan drops the group into KNOWN NOISE with "marked <date>", and UNMARK restores it to SIGNAL; OPEN JOURNAL opens journalctl -p 3 -b in a foot terminal; a failed unit's RESTART on a real unit (systemctl restart via the armed key) lands the re-probe on the act line. Phase 9b additions: MARK EXPECTED on ratio's real unexpected listeners (postgres :5432, docker :8080 — the standing curation material) dims them into the expected list and UNMARK restores them; a CPU-mode press (e.g. BAL·PWR) writes the EPP hint and the row re-reads it; KILL renders disabled (insensitive, tooltip) on a session-critical name in top memory; on velox the battery row shows NO SET 80% key (cros_ec knob absent — correct withholding, not a bug). +Phase 11 additions: glyph left-click opens the console and a second click closes it; right-click still toggles the btop scratchpad; the glyph color matches the board's worst lever-less diagnostic (open the console and check the watch items against the glyph tint); after fixing/curating a diagnostic, the glyph follows within one 30-min scan (or immediately after a manual =maint scan=); on velox in person — battery % + charging icon in the bar, collapsed bar keeps the battery glyph, low-charge red (drain below 15% or temporarily raise battery_low_pct in the TOML); after velox's reboot confirm systemctl --user list-timers shows both maint timers scheduled. Phase 10 additions: results-wall readability during a real doctor run (press CLEAN UP on the live board — every step should appear as a running line the moment it starts, resolve in place with the result, and the re-probed truth should follow; the wall should feel like watching the run, not reading a report afterward); COPY puts the same lines the CLI wall prints on the clipboard, paste somewhere to check; HIDE collapses the wall to its header and SHOW restores it with the scroll position at the bottom; REVIEW & FIX reads right at the moment of use (each row: what's wrong, the maint fix usage, and either a FIX key or "per-item — on its subpanel"); RECLAIM on a degraded disk row arms with the first step + "(+N more steps)" and the fire streams every step to the wall individually; a guard-tripped UPDATE fired from the REVIEW roster re-arms with the override wording and the second press actually runs (the token fix — verify the loop terminates). - -*** TODO Flip the spec to IMPLEMENTED -When all phases land: spec keyword → IMPLEMENTED + dated history line naming the closing commits + Status mirror. The lifecycle loop closes here — do not skip. +Phase 11b additions: the fidelity eyeball — open the live console beside the settled E5 prototype (headless-Chrome render or the browser) and judge whether the board finally reads as the same instrument: big-number typography, bar fills with the gold threshold tick on the cache card, the scrub/keyring/topgrade rings, status chips, the lit CPU-mode segment, selector count chips + severity left borders, right-aligned attention/ok/fixable/watch header; press an EPP segment on the card (e.g. BAL·PWR) and confirm it arms then fires exactly like the old digest control; on the bad fixture (MAINT_PANEL_FIXTURE=bad maint panel) check a long readout (the BACKUPS card) ellipsizes with a tooltip carrying the full value. +*** 2026-07-08 Wed @ 06:31:00 -0500 Fixed TOPGRADE remedy failure found in testing (dotfiles 9b5384f) +Craig's first testing find: TOPGRADE exited 2 on every launch — topgrade 17.6.1 names the git step git_repos and the remedy passed --disable git, rejected by clap before any step ran. Fixed red-first (test pins git_repos), full make test green, dry-run on the installed binary confirms the step name lands and the repo-rebasing step stays excluded. Velox pulled. RE-TEST: press TOPGRADE again (it's guard-armed on ratio's pending mesa set — TTY is still the safe path for the mesa run). ** TODO [#D] Maintenance console vNext :feature: Deferred from the [[file:docs/specs/2026-07-07-maintenance-console-spec.org][spec]] (v1 ships determinate remedies only): AI/workflow assistance on read-only metrics (the vLater tier — failed-unit diagnosis, journal-error fix suggestions, OOM investigation); SIGKILL escalation for TERM-survivors on the KILL lever; live streaming meters on evidence rows where a count could be a level. +** TODO [#D] Retention-repair remedy unreachable from the panel GUI :bug:dotfiles: +snapshot_retention_repair (WRITE SANE LIMITS — =snapper -c <config> set-config TIMELINE_CREATE=yes TIMELINE_LIMIT_*=<TOML values>=) sits in the maint remedy table but is wired into no GUI layer (no reference in panel.py, viewmodel.py, or gui.py) — it's reachable only via =maint doctor review= / =maint fix=. Worse, the REVIEW & FIX roster's wording for item-bearing remedies says "per-item — on its subpanel", which for this remedy points at a key that doesn't exist. Found 2026-07-08 while walking the SNAPSHOTS subpanel with Craig. + +Two fix shapes, decide at work time: (1) wire a digest key — e.g. on the timeline row when the config's installed TIMELINE_LIMIT_* drift from the TOML values (needs the probe to read the installed limits for comparison); or (2) keep it CLI-only deliberately and special-case the roster wording for remedies with no panel key ("CLI only — maint fix"). Option 2 is a few lines; option 1 makes limit drift visible on the board, which is the 2026-05-26 pile-up's root cause. Severity minor × rare edge = P4 per the bug matrix. + +** TODO [#C] minimal tier zsh login PATH gap :bug:dotfiles:quick:solo: +The 2026-07-08 fix gave common/ a .zprofile (zsh logins never read .profile, so ~/.local/bin was missing from TTY/ssh PATH — dotfiles 39cef40). The minimal/ tier has the same gap: it carries .zshrc and .profile but no .zprofile, so headless installs (DESKTOP_ENV=none stows minimal/ instead of common/) still get zsh logins without ~/.local/bin. Add a matching minimal/.zprofile (PATH-only prepend, same rationale comment). Verify: fresh minimal stow, zsh -lc 'echo $PATH' shows ~/.local/bin first. The installer itself is clean — it writes no shell profiles; the stow layer owns this. + +** TODO [#C] zfs base VM image build failure: ZFS DKMS module missing :bug:zfs: +=FS_PROFILE=zfs make test-vm-base= fails inside the VM at initramfs time: archangel reports "ZFS module not found! DKMS build may have failed" against the installed kernel (linux-lts 6.18.38 at the 2026-07-08 attempt). Likely a kernel/OpenZFS version skew — the archzfs zfs-dkms may not support 6.18 yet. Consequences: the maint scenario harness's zfs lane (Phase 12) is filtered but unexercised, and a real zfs bare-metal install via archangel would plausibly hit the same wall. Diagnosis needs the in-VM pacman/DKMS logs (rerun create-base-vm.sh with =--keep=-style access or pull /var/log from the ISO session before teardown). Priority per the bug matrix: Major severity (zfs install path broken) × some-users-sometimes = P3. When fixed, run =FS_PROFILE=zfs bash scripts/testing/run-maint-scenarios.sh --list= and add zfs scenario files (zpool scrub / autotrim / snapshot destroy) to the harness. + ** TODO [#C] Consistent keybinding family for the panel console :feature:hyprland: Consider putting every panel (net, bluetooth, audio, timer, and the coming maintenance console) on one consistent chord family — a shared modifier set (Super+Shift, Control+Alt, or similar) plus a mnemonic letter per panel (N/B/A/T/M). Today the panels open via waybar clicks only; a uniform chord family makes them keyboard-reachable and predictable. Watch for collisions with existing binds: Super+Shift+A is already PTT toggle, and the hold-to-talk grave bind is load-bearing. Decide the family, audit current hyprland binds for conflicts, wire via the dotfiles hyprland config, and document in the keybind reference. Both machines (velox can't QMK-remap, so chords must work on a plain laptop keyboard). |