diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/post-rebuild-check | 85 |
1 files changed, 77 insertions, 8 deletions
diff --git a/scripts/post-rebuild-check b/scripts/post-rebuild-check index 8807f85..c18ae8f 100755 --- a/scripts/post-rebuild-check +++ b/scripts/post-rebuild-check @@ -21,6 +21,10 @@ # 5. signal-cli holds no registered account (velox lost its # registration, and because agent-text relays into this machine, # that silently broke paging for the WHOLE fleet) +# 6. every NTP source is named by hostname (a wrong clock fails the +# DoT/DNSSEC validation this machine's DNS runs on, so nothing +# resolves -- including the NTP pool that would fix the clock; velox +# deadlocked exactly this way 2026-08-19 and needed a second device) # # The .gitignore rule in check 4 is what scopes it: a tooling path is only # expected where the project's own .gitignore names it, so a project that @@ -50,6 +54,8 @@ # ~/.dotfiles) # PRC_SIGNAL_ACCOUNTS signal-cli listAccounts output; "" = no account, # the special value MISSING = binary absent +# PRC_NTP_SOURCES newline list of configured NTP server addresses; +# the special value MISSING = no NTP daemon active # PRC_SYSTEMCTL path to the systemctl binary (a fake, under test) # PRC_SYSTEMCTL_TIMEOUT seconds to allow each systemctl call (default 5) # @@ -61,9 +67,10 @@ usage() { cat <<'EOF' post-rebuild-check - verify a rebuilt machine is actually finished -Runs the five checks that caught velox's 2026-08 reinstall gaps: failed +Runs the six checks that caught velox's 2026-08 reinstall gaps: failed units, present-but-inert user units, orphaned *.example configs, missing -per-project tooling state, and the signal-cli registration. +per-project tooling state, the signal-cli registration, and whether time +sync can recover from a wrong clock without DNS. Usage: post-rebuild-check [--help] @@ -84,6 +91,7 @@ TOTAL_FINDINGS=0 CHECK_FINDINGS=0 FINDING_LINES="" signal_missing="" +ntp_missing="" # Every systemctl call is bounded. A wedged user manager spins and answers # nothing -- seen live on velox 2026-08-17, where `is-enabled`, `cat`, and @@ -179,7 +187,7 @@ while IFS= read -r line; do unit=${line#*:} finding "$scope unit failed: $unit" done < "$STAGE" -report "check 1/5: failed units" +report "check 1/6: failed units" # --- 2. user unit files present but not enabled --------------------------- @@ -269,7 +277,7 @@ while read -r name state; do esac finding "unit file present but not enabled: $name ($state)" done < "$STAGE" -report "check 2/5: unit files" +report "check 2/6: unit files" # --- 3. *.example files whose real sibling is missing --------------------- @@ -314,7 +322,7 @@ while IFS= read -r root; do [ -e "${ex%.example}" ] || finding "example without its real file: $ex" done < "$WORK/examples" done < "$WORK/roots" -report "check 3/5: local files" +report "check 3/6: local files" # --- 4. gitignore-mode projects missing their tooling --------------------- @@ -368,7 +376,7 @@ todo.org todo\.org inbox inbox EOF done < "$WORK/projects" -report "check 4/5: project tooling" +report "check 4/6: project tooling" # --- 5. signal-cli registration ------------------------------------------- @@ -395,7 +403,68 @@ if [ "$signal_missing" = 1 ]; then elif [ -z "$signal_missing" ] && [ -z "$accounts" ]; then finding "no signal account registered — agent-text relays into this machine, so paging breaks for the whole fleet" fi -report "check 5/5: signal registration" +report "check 5/6: signal registration" + +# --- 6. NTP can recover a wrong clock without DNS ------------------------- +# +# The clock/DNS bootstrap deadlock. This machine resolves through DNSOverTLS +# with DNSSEC, and both validate against the wall clock, so a boot with a +# wrong clock resolves nothing at all. If every configured NTP source is named +# by hostname, the daemon that would correct the clock needs the DNS the clock +# is breaking, and the machine cannot recover without a second device -- +# which is exactly what happened on velox 2026-08-19. One source addressed by +# IP breaks the cycle, so that is what this check looks for. + +# True when the argument is an address rather than a name. An address needs no +# resolver, which is the whole property being checked. +is_ip_literal() { + case "$1" in + "") return 1 ;; + *:*) case "$1" in *[!0-9A-Fa-f:]*) return 1 ;; esac + return 0 ;; + *[!0-9.]*) return 1 ;; + *.*) return 0 ;; + esac + return 1 +} + +if [ -n "${PRC_NTP_SOURCES+set}" ]; then + ntp_sources=$PRC_NTP_SOURCES + if [ "$ntp_sources" = "MISSING" ]; then + ntp_sources="" + ntp_missing=1 + fi +elif sctl is-active chronyd >/dev/null 2>&1; then + # Both the main file and any drop-in: the IP-addressed source belongs in a + # drop-in, so reading only chrony.conf would miss every correct machine. + ntp_sources=$(cat /etc/chrony.conf /etc/chrony.d/*.conf 2>/dev/null \ + | awk '$1 == "server" || $1 == "pool" { print $2 }') +elif sctl is-active systemd-timesyncd >/dev/null 2>&1; then + ntp_sources=$(awk -F= '/^[[:space:]]*NTP=/ { print $2 }' \ + /etc/systemd/timesyncd.conf 2>/dev/null | tr ' ' '\n') +else + ntp_sources="" + ntp_missing=1 +fi + +if [ "$ntp_missing" = 1 ]; then + finding "no NTP implementation is active — nothing corrects the clock, and a wrong clock takes DNS down with it" +elif [ -z "$ntp_sources" ]; then + finding "no NTP sources are configured — nothing was checked, and nothing corrects the clock" +else + ntp_has_literal="" + stage "$ntp_sources" + while IFS= read -r src_addr; do + [ -z "$src_addr" ] && continue + if is_ip_literal "$src_addr"; then + ntp_has_literal=1 + fi + done < "$STAGE" + if [ -z "$ntp_has_literal" ]; then + finding "every NTP source is named by hostname — a wrong clock breaks DNS, so nothing can resolve them and the clock stays wrong" + fi +fi +report "check 6/6: NTP bootstrap" # --- summary -------------------------------------------------------------- @@ -403,5 +472,5 @@ if [ "$TOTAL_FINDINGS" -eq 0 ]; then echo "all checks clean" exit 0 fi -echo "$TOTAL_FINDINGS finding(s) across 5 checks" +echo "$TOTAL_FINDINGS finding(s) across 6 checks" exit 1 |