path: root/scripts/post-rebuild-check

#!/bin/sh
# SPDX-License-Identifier: GPL-3.0-or-later
# post-rebuild-check - the five checks a rebuilt machine actually needs.
#
# A rebuilt machine looks finished and isn't. Five gaps surfaced on velox
# within two days of the 2026-08-13 reinstall, and three of them LOOKED
# fine: a stowed unit file, an enabled timer, a present git clone. Each
# check below is cheap and turns a silent no-op into a visible line:
#
#   1. failed systemd units, user and system scope (calendar-sync failed
#      every 15 minutes for two days with nobody watching)
#   2. user unit files present but not enabled (roam-sync and
#      signal-receive came back linked and inert -- a unit file being
#      present is not the same as running)
#   3. tracked *.example files whose real sibling is missing (three
#      *.local.el were gone on velox; the .example survives in git,
#      the real file never does)
#   4. gitignore-mode projects missing tooling paths their own .gitignore
#      names (a reinstall drops every such project's untracked working
#      state -- 374 files in .emacs.d's case -- and nothing carries it)
#   5. signal-cli holds no registered account (velox lost its
#      registration, and because agent-text relays into this machine,
#      that silently broke paging for the WHOLE fleet)
#
# The .gitignore rule in check 4 is what scopes it: a tooling path is only
# expected where the project's own .gitignore names it, so a project that
# never had a todo.org never flags. The ignore file is the project's own
# record of what it is supposed to hold untracked.
#
# EVERY PROBE FAILS CLOSED. A check that cannot run reports a finding, never
# a pass. This matters more here than anywhere else in the script: the whole
# point is catching silent no-ops, so a silent no-op in the checker would be
# the worst possible defect. `systemctl --user` exits 1 with empty output
# when there is no user bus -- over ssh, from cron, under sudo, on a TTY
# before the graphical session starts -- and reading that as "no failed
# units" would report a machine as healthy exactly when nothing was checked.
#
# Exit 0 when every check is clean, 1 when any check found something,
# 2 on usage error.
#
# Test seams (env; for each, set-but-empty means "the probe ran and found
# nothing", unset means "run the real probe"):
#   PRC_FAILED_UNITS      newline list of "scope:unit" (scope user|system)
#   PRC_UNIT_STATES       newline list of "unit-file state" replacing the
#                         user-unit-dir enumeration + is-enabled calls
#   PRC_LOCAL_SCAN_ROOTS  NEWLINE-separated roots for the *.example scan
#                         (default: ~/.emacs.d ~/.dotfiles)
#   PRC_PROJECT_ROOTS     NEWLINE-separated project dirs for check 4
#                         (default: ~/code/* ~/projects/* ~/.emacs.d
#                         ~/.dotfiles)
#   PRC_SIGNAL_ACCOUNTS   signal-cli listAccounts output; "" = no account,
#                         the special value MISSING = binary absent
#   PRC_SYSTEMCTL         path to the systemctl binary (a fake, under test)
#   PRC_SYSTEMCTL_TIMEOUT seconds to allow each systemctl call (default 5)
#
# Roots are newline-separated, not space-separated, because a POSIX
# `for root in $var` splits on spaces and turns one real directory into
# several imaginary missing ones.

usage() {
    cat <<'EOF'
post-rebuild-check - verify a rebuilt machine is actually finished

Runs the five checks that caught velox's 2026-08 reinstall gaps: failed
units, present-but-inert user units, orphaned *.example configs, missing
per-project tooling state, and the signal-cli registration.

Usage: post-rebuild-check [--help]

Exit 0 when every check is clean, 1 when any check found something.
Every probe fails closed: a check that cannot run is a finding, not a pass.
EOF
}

case "${1:-}" in
    --help|-h) usage; exit 0 ;;
    "") ;;
    *) echo "post-rebuild-check: unknown argument: $1" >&2; usage >&2; exit 2 ;;
esac

# Own the internal flags rather than inheriting them, so a caller's unrelated
# variable of the same name cannot manufacture or mask a finding.
TOTAL_FINDINGS=0
CHECK_FINDINGS=0
FINDING_LINES=""
signal_missing=""

# Every systemctl call is bounded. A wedged user manager spins and answers
# nothing -- seen live on velox 2026-08-17, where `is-enabled`, `cat`, and
# `list-unit-files` all hung while `list-units` still returned. Unbounded, this
# script would hang on the first unit and never reach the remaining checks,
# which is a worse failure than reporting nothing: a check that hangs is its
# own outage, and the machine most in need of checking is the one it hangs on.
# A timeout yields empty output and a non-zero status, and both are already
# handled as findings, so bounding the call is all that is needed to fail closed.
SCTL_TIMEOUT=${PRC_SYSTEMCTL_TIMEOUT:-5}
SYSTEMCTL=${PRC_SYSTEMCTL:-systemctl}

sctl() {
    if command -v timeout >/dev/null 2>&1; then
        timeout "$SCTL_TIMEOUT" "$SYSTEMCTL" "$@"
    else
        # Say so rather than dropping the bound silently: without timeout a
        # wedged manager hangs this run indefinitely, and the whole point of
        # the bound is that a check which hangs reports nothing at all.
        [ -n "${sctl_unbounded_warned:-}" ] || {
            echo "post-rebuild-check: timeout(1) not found — systemctl calls are UNBOUNDED and may hang" >&2
            sctl_unbounded_warned=1
        }
        "$SYSTEMCTL" "$@"
    fi
}

WORK=${TMPDIR:-/tmp}/.post-rebuild-check.$$
if ! mkdir "$WORK" 2>/dev/null; then
    # Every check stages its input through a file in here. Without it each
    # loop would read nothing and every check would come back clean, which is
    # the one failure this script must never produce.
    echo "post-rebuild-check: cannot create a work directory under ${TMPDIR:-/tmp}" >&2
    echo "  nothing was checked; this is not a pass" >&2
    exit 1
fi
trap 'rm -rf "$WORK"' EXIT HUP INT TERM

STAGE="$WORK/stage"

finding() {
    CHECK_FINDINGS=$((CHECK_FINDINGS + 1))
    TOTAL_FINDINGS=$((TOTAL_FINDINGS + 1))
    FINDING_LINES="${FINDING_LINES}  DEVIATION: $1
"
}

# Print the check's one visible line, then its findings. The visible line
# is the point: a silent no-op is exactly what let the gaps sit unseen.
report() {
    if [ "$CHECK_FINDINGS" -eq 0 ]; then
        echo "$1 — ok"
    else
        echo "$1 — $CHECK_FINDINGS finding(s)"
        printf '%s' "$FINDING_LINES"
    fi
    CHECK_FINDINGS=0
    FINDING_LINES=""
}

# Stage a value into $STAGE for the read loops. A failed write is fatal for
# the same reason a missing work directory is.
stage() {
    if ! printf '%s\n' "$1" > "$STAGE" 2>/dev/null; then
        echo "post-rebuild-check: cannot write $STAGE" >&2
        echo "  nothing was checked; this is not a pass" >&2
        exit 1
    fi
}

# --- 1. failed units ------------------------------------------------------

if [ -n "${PRC_FAILED_UNITS+set}" ]; then
    failed=$PRC_FAILED_UNITS
else
    failed=""
    if user_out=$(sctl --user list-units --state=failed --no-legend --plain 2>/dev/null); then
        failed=$(printf '%s' "$user_out" | awk 'NF {print "user:"$1}')
    else
        finding "could not query user units (no user bus?) — nothing was checked in this scope"
    fi
    if sys_out=$(sctl list-units --state=failed --no-legend --plain 2>/dev/null); then
        failed="$failed
$(printf '%s' "$sys_out" | awk 'NF {print "system:"$1}')"
    else
        finding "could not query system units — nothing was checked in this scope"
    fi
fi
stage "$failed"
while IFS= read -r line; do
    [ -n "$line" ] || continue
    scope=${line%%:*}
    unit=${line#*:}
    finding "$scope unit failed: $unit"
done < "$STAGE"
report "check 1/5: failed units"

# --- 2. user unit files present but not enabled ---------------------------

if [ -n "${PRC_UNIT_STATES+set}" ]; then
    states=$PRC_UNIT_STATES
else
    states=""
    unit_dir="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
    if [ ! -d "$unit_dir" ]; then
        finding "no user unit directory at $unit_dir — nothing was checked"
    else
        for f in "$unit_dir"/*.timer "$unit_dir"/*.service; do
            # -L as well as -e: a stow symlink whose target moved in the
            # rebuild is exactly the "looked fine" case this check is for,
            # and -e is false for a broken link.
            [ -e "$f" ] || [ -L "$f" ] || continue
            name=$(basename "$f")
            # A link with nothing behind it is its own finding, decided on the
            # filesystem rather than from systemd. `is-enabled` calls a
            # dangling link "not-found" -- the same answer it gives for a unit
            # that was never installed -- so routing this through the state
            # table below would drop it silently.
            if [ -L "$f" ] && [ ! -e "$f" ]; then
                finding "stowed unit file points at a missing target: $name"
                continue
            fi
            # is-enabled exits non-zero AND prints a state for disabled and
            # linked, so the exit code cannot distinguish "this unit is
            # disabled" from "the query failed". The output can: a real answer
            # is always a word. Empty means no answer, which is a finding
            # rather than a silent skip -- with no user bus (ssh, cron, sudo,
            # a TTY before the graphical session) every unit answers empty,
            # and treating that as unknown-so-ignore would pass the machine
            # while reading nothing at all.
            #
            # No separate bus probe: `is-system-running` and
            # `show-environment` both block here, and a check that can hang is
            # its own outage.
            state=$(sctl --user is-enabled "$name" 2>/dev/null)
            if [ -z "$state" ]; then
                finding "could not read the enablement state of $name — it was not checked"
                continue
            fi
            states="${states}${name} ${state}
"
        done
    fi
fi
stage "$states"
# A second copy for the sibling-timer lookup below, so the awk that reads it
# is never the same open file as the loop reading it.
cp "$STAGE" "$WORK/states" 2>/dev/null || {
    echo "post-rebuild-check: cannot write $WORK/states" >&2
    echo "  nothing was checked; this is not a pass" >&2; exit 1; }
while read -r name state; do
    [ -n "$name" ] || continue
    case "$state" in
        disabled|linked) ;;
        *) continue ;;
    esac
    # A timer-activated service is SUPPOSED to sit linked-and-not-enabled:
    # the timer owns activation, and enabling the service as well would run
    # it at boot on top of its schedule. So a service is suppressed only when
    # its sibling timer can actually start it (enabled), or when the timer is
    # itself inert and therefore the finding already -- reporting both would
    # name one gap twice. A masked, static, or not-found timer starts
    # nothing, so the service beneath it is as dead as one with no timer.
    case "$name" in
        *.service)
            timer="${name%.service}.timer"
            tstate=$(awk -v t="$timer" '$1 == t {print $2; exit}' "$WORK/states")
            # enabled-runtime (enabled until reboot) and generated (something
            # produced and installed it) are live activation paths, so the
            # service under one is being started and is not a finding.
            # disabled and linked suppress for a different reason: the timer
            # is then the finding itself, reported in its own right.
            #
            # "indirect" deliberately does NOT suppress. It means the unit
            # file itself is not enabled, only that some Also= relative might
            # be, so nothing here is known to start the service. The
            # fail-closed rule says the uncertain case flags.
            case "$tstate" in
                enabled|enabled-runtime|generated) continue ;;
                disabled|linked) continue ;;
            esac
            ;;
    esac
    finding "unit file present but not enabled: $name ($state)"
done < "$STAGE"
report "check 2/5: unit files"

# --- 3. *.example files whose real sibling is missing ---------------------

if [ -n "${PRC_LOCAL_SCAN_ROOTS+set}" ]; then
    scan_roots=$PRC_LOCAL_SCAN_ROOTS
else
    scan_roots="$HOME/.emacs.d
$HOME/.dotfiles"
fi
printf '%s\n' "$scan_roots" > "$WORK/roots" 2>/dev/null || {
    echo "post-rebuild-check: cannot write $WORK/roots" >&2; exit 1; }
while IFS= read -r root; do
    [ -n "$root" ] || continue
    if [ ! -d "$root" ]; then
        finding "scan root missing: $root"
        continue
    fi
    # Vendored package trees ship their own .example docs; those belong to
    # the package, not to this machine, so they are noise in front of the
    # real findings this check exists for.
    #
    # -prune, not -not -path: the latter filters find's OUTPUT while still
    # descending, so an unreadable directory inside a tree we deliberately
    # ignore would set find's exit status and be reported as an unscanned
    # part of the root. Pruning means those trees are never entered, so the
    # exit status only reflects places this check actually wanted to read.
    #
    # That status matters: find exits non-zero when it cannot descend
    # somewhere, having printed only what it could reach. Discarding it would
    # hide every orphan under an unreadable directory behind a clean "ok",
    # which is the defect this script exists to catch.
    if ! find "$root" \
         \( -name .git -o -name elpa -o -name straight \
            -o -name node_modules -o -name .venv \) -prune \
         -o -name '*.example' -print > "$WORK/examples" 2>/dev/null; then
        finding "could not fully scan $root — part of it was not checked"
    fi
    while IFS= read -r ex; do
        [ -n "$ex" ] || continue
        # -e, so a sibling that exists only as a dangling symlink counts as
        # missing. It is not a config the machine can read.
        [ -e "${ex%.example}" ] || finding "example without its real file: $ex"
    done < "$WORK/examples"
done < "$WORK/roots"
report "check 3/5: local files"

# --- 4. gitignore-mode projects missing their tooling ---------------------

if [ -n "${PRC_PROJECT_ROOTS+set}" ]; then
    projects=$PRC_PROJECT_ROOTS
else
    projects=$(ls -d "$HOME"/code/*/ "$HOME"/projects/*/ 2>/dev/null; \
               printf '%s\n%s\n' "$HOME/.emacs.d" "$HOME/.dotfiles")
fi
printf '%s\n' "$projects" > "$WORK/projects" 2>/dev/null || {
    echo "post-rebuild-check: cannot write $WORK/projects" >&2; exit 1; }
# CLAUDE.md is deliberately absent from this set. It is seed-only --
# install-lang writes it once and the project owns it afterward -- so most
# projects legitimately never have one, and ratio shows the identical
# absences in the identical projects. That match is what proves it is the
# steady state rather than reinstall drift, and flagging it would put nine
# standing findings in front of every real one.
#
# The list is fed to the inner loop straight from a heredoc rather than
# staged through a file. It is a constant, so a file bought nothing and cost
# a fifth unguarded write: had it failed (a full tmpfs, say) the inner loop
# would read nothing and every project would pass silently, which is the one
# outcome this script must never produce. The heredoc is the inner loop's own
# stdin and leaves the outer loop's redirect alone.
while IFS= read -r proj; do
    [ -n "$proj" ] || continue
    proj=${proj%/}
    # -e not -d: in a worktree or submodule .git is a file naming the real
    # gitdir, and a -d test would skip those projects silently.
    [ -e "$proj/.git" ] || continue
    [ -f "$proj/.gitignore" ] || continue
    while read -r disk pattern; do
        # Both the anchored (/.ai/) and unanchored (.ai/) ignore styles exist
        # across the fleet; the sweep-gitignore audit hit exactly that split.
        #
        # grep exits 1 for no-match and 2 for an error, so the two are told
        # apart rather than both read as "the ignore file does not name this".
        # An unreadable .gitignore would otherwise pass the whole project.
        grep -Eq "^/?${pattern}/?\$" "$proj/.gitignore" 2>/dev/null
        case $? in
            0) [ -e "$proj/$disk" ] \
                   || finding "$proj: .gitignore names $disk but it is missing on disk" ;;
            1) ;;
            *) finding "$proj: could not read .gitignore — the project was not checked"
               break ;;
        esac
    done <<'EOF'
.ai \.ai
.claude \.claude
todo.org todo\.org
inbox inbox
EOF
done < "$WORK/projects"
report "check 4/5: project tooling"

# --- 5. signal-cli registration -------------------------------------------

if [ -n "${PRC_SIGNAL_ACCOUNTS+set}" ]; then
    accounts=$PRC_SIGNAL_ACCOUNTS
    if [ "$accounts" = "MISSING" ]; then
        accounts=""
        signal_missing=1
    fi
else
    if command -v signal-cli >/dev/null 2>&1; then
        if ! accounts=$(signal-cli listAccounts 2>/dev/null); then
            accounts=""
            finding "signal-cli listAccounts failed — the registration was not checked"
            signal_missing=skip
        fi
    else
        accounts=""
        signal_missing=1
    fi
fi
if [ "$signal_missing" = 1 ]; then
    finding "signal-cli is not installed — paging relies on it fleet-wide"
elif [ -z "$signal_missing" ] && [ -z "$accounts" ]; then
    finding "no signal account registered — agent-text relays into this machine, so paging breaks for the whole fleet"
fi
report "check 5/5: signal registration"

# --- summary --------------------------------------------------------------

if [ "$TOTAL_FINDINGS" -eq 0 ]; then
    echo "all checks clean"
    exit 0
fi
echo "$TOTAL_FINDINGS finding(s) across 5 checks"
exit 1