diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/testing/lib/maint-scenario.sh | 62 | ||||
| -rw-r--r-- | scripts/testing/lib/run-common.sh | 61 | ||||
| -rw-r--r-- | scripts/testing/run-maint-nspawn.sh | 60 | ||||
| -rw-r--r-- | scripts/testing/run-maint-scenarios.sh | 65 | ||||
| -rwxr-xr-x | scripts/testing/run-test-baremetal.sh | 65 | ||||
| -rwxr-xr-x | scripts/testing/run-test.sh | 82 | ||||
| -rw-r--r-- | scripts/wipedisk | 76 | ||||
| -rwxr-xr-x | scripts/zfs-replicate | 25 |
8 files changed, 261 insertions, 235 deletions
diff --git a/scripts/testing/lib/maint-scenario.sh b/scripts/testing/lib/maint-scenario.sh new file mode 100644 index 0000000..e7fcdda --- /dev/null +++ b/scripts/testing/lib/maint-scenario.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Shared contract and execution helpers for maint scenario transports. + +# Extract one declared variable from a scenario file in a clean subshell. +_scenario_var() { # <file> <varname> + bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ + _probe "$1" "$2" 2>/dev/null +} + +_validate_scenario() { # <file> -> fatal on contract violation + local f="$1" base probe_err + base=$(basename "$f") + probe_err=$(bash -c 'set -eu; source "$1" + : "${SCENARIO_DESC:?missing SCENARIO_DESC}" + : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" + : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" + for p in $SCENARIO_PROFILES; do + case "$p" in btrfs|zfs|any) ;; *) + echo "bad profile token: $p" >&2; exit 1 ;; + esac + done + declare -f scenario_break scenario_fix scenario_assert >/dev/null \ + || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ + _probe "$f" 2>&1 >/dev/null) \ + || fatal "scenario contract violation in $base: $probe_err" +} + +run_scenario() { # <index> + local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" + section "Scenario: $name — ${S_DESCS[$1]}" + # shellcheck disable=SC1090 + source "$f" + local ok=true + step "break" + if ! scenario_break; then + error "$name: break step failed" + ok=false + fi + if [ "$ok" = "true" ]; then + step "fix" + if ! scenario_fix; then + error "$name: fix step failed" + ok=false + fi + fi + if [ "$ok" = "true" ]; then + step "assert" + if ! scenario_assert; then + error "$name: post-state assertion failed" + ok=false + fi + fi + unset -f scenario_break scenario_fix scenario_assert + unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES + if [ "$ok" = "true" ]; then + success "PASS: $name" + PASS+=("$name") + else + error "FAIL: $name" + FAIL+=("$name") + fi +} diff --git a/scripts/testing/lib/run-common.sh b/scripts/testing/lib/run-common.sh new file mode 100644 index 0000000..b5696db --- /dev/null +++ b/scripts/testing/lib/run-common.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Transport-independent polling and reporting for archsetup test runners. + +archsetup_process_running() { # <transport-function> + local transport="$1" + "$transport" "pgrep -f '[b]ash archsetup' > /dev/null" 2>/dev/null +} + +wait_for_archsetup() { # <transport> <max-polls> <seconds> <progress-every> [callback] + local transport="$1" max_polls="$2" poll_seconds="$3" + local progress_every="$4" progress_callback="${5:-}" + ARCHSETUP_POLL_COUNT=0 + + while [ "$ARCHSETUP_POLL_COUNT" -lt "$max_polls" ]; do + archsetup_process_running "$transport" || return 0 + sleep "$poll_seconds" + ARCHSETUP_POLL_COUNT=$((ARCHSETUP_POLL_COUNT + 1)) + if [ "$progress_every" -gt 0 ] \ + && [ $((ARCHSETUP_POLL_COUNT % progress_every)) -eq 0 ] \ + && [ -n "$progress_callback" ]; then + "$progress_callback" "$ARCHSETUP_POLL_COUNT" + fi + done + return 124 +} + +write_archsetup_test_report() { + local file="$1" title="$2" test_id="$3" method="$4" context="$5" + local completed="$6" passed="$7" count_passed="$8" count_failed="$9" + shift 9 + local count_warnings="$1" artifacts="$2" validation="FAILED" + [ "$passed" = "true" ] && validation="PASSED" + + { + echo "========================================" + echo "$title" + echo "========================================" + echo + echo "Test ID: $test_id" + echo "Date: $(date +'%Y-%m-%d %H:%M:%S')" + echo "Test Method: $method" + if [ -n "$context" ]; then + echo + printf '%s\n' "$context" + fi + echo + echo "Results:" + echo " ArchSetup Completed: $completed (completion marker, not the installer's exit code)" + echo " Validation: $validation" + echo + echo "Validation Summary:" + echo " Passed: $count_passed" + echo " Failed: $count_failed" + echo " Warnings: $count_warnings" + if [ -n "$artifacts" ]; then + echo + printf '%s\n' "$artifacts" + fi + echo + } > "$file" +} diff --git a/scripts/testing/run-maint-nspawn.sh b/scripts/testing/run-maint-nspawn.sh index 04d4a32..dcc7c16 100644 --- a/scripts/testing/run-maint-nspawn.sh +++ b/scripts/testing/run-maint-nspawn.sh @@ -32,6 +32,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "$SCRIPT_DIR/lib/logging.sh" +source "$SCRIPT_DIR/lib/maint-scenario.sh" SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" @@ -63,29 +64,6 @@ done S_FILES=() S_NAMES=() S_DESCS=() -_scenario_var() { # <file> <varname> - bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ - _probe "$1" "$2" 2>/dev/null -} - -_validate_scenario() { # <file> -> fatal on contract violation - local f="$1" base probe_err - base=$(basename "$f") - probe_err=$(bash -c 'set -eu; source "$1" - : "${SCENARIO_DESC:?missing SCENARIO_DESC}" - : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" - : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" - for p in $SCENARIO_PROFILES; do - case "$p" in btrfs|zfs|any) ;; *) - echo "bad profile token: $p" >&2; exit 1 ;; - esac - done - declare -f scenario_break scenario_fix scenario_assert >/dev/null \ - || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ - _probe "$f" 2>&1 >/dev/null) \ - || fatal "scenario contract violation in $base: $probe_err" -} - build_plan() { [ -d "$SCENARIO_DIR" ] || fatal "scenario dir not found: $SCENARIO_DIR" local f base @@ -212,42 +190,6 @@ success "Container ready" PASS=() FAIL=() -run_scenario() { # <index> - local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" - section "Scenario: $name — ${S_DESCS[$1]}" - # shellcheck disable=SC1090 - source "$f" - local ok=true - step "break" - if ! scenario_break; then - error "$name: break step failed" - ok=false - fi - if [ "$ok" = "true" ]; then - step "fix" - if ! scenario_fix; then - error "$name: fix step failed" - ok=false - fi - fi - if [ "$ok" = "true" ]; then - step "assert" - if ! scenario_assert; then - error "$name: post-state assertion failed" - ok=false - fi - fi - unset -f scenario_break scenario_fix scenario_assert - unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES - if [ "$ok" = "true" ]; then - success "PASS: $name" - PASS+=("$name") - else - error "FAIL: $name" - FAIL+=("$name") - fi -} - for i in "${!S_FILES[@]}"; do run_scenario "$i" done diff --git a/scripts/testing/run-maint-scenarios.sh b/scripts/testing/run-maint-scenarios.sh index 448f6d9..17481e9 100644 --- a/scripts/testing/run-maint-scenarios.sh +++ b/scripts/testing/run-maint-scenarios.sh @@ -35,6 +35,7 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "$SCRIPT_DIR/lib/logging.sh" source "$SCRIPT_DIR/lib/vm-utils.sh" +source "$SCRIPT_DIR/lib/maint-scenario.sh" SCENARIO_DIR="${MAINT_SCENARIO_DIR:-$SCRIPT_DIR/maint-scenarios}" MAINT_SRC="${MAINT_SRC:-$HOME/.dotfiles/maint/src/maint}" @@ -72,32 +73,6 @@ S_FILES=() S_NAMES=() S_DESCS=() S_GROUPS=() # Groups in first-appearance order. GROUP_ORDER=() -# Extract one declared var from a scenario file by sourcing it in a clean -# subshell. Scenario files must only define vars + functions; any top-level -# command fails here (no helpers exist), which is the contract. -_scenario_var() { # <file> <varname> - bash -c 'set -eu; source "$1" || exit 9; eval "printf %s \"\${$2-}\""' \ - _probe "$1" "$2" 2>/dev/null -} - -_validate_scenario() { # <file> -> fatal on contract violation - local f="$1" base probe_err - base=$(basename "$f") - probe_err=$(bash -c 'set -eu; source "$1" - : "${SCENARIO_DESC:?missing SCENARIO_DESC}" - : "${SCENARIO_GROUP:?missing SCENARIO_GROUP}" - : "${SCENARIO_PROFILES:?missing SCENARIO_PROFILES}" - for p in $SCENARIO_PROFILES; do - case "$p" in btrfs|zfs|any) ;; *) - echo "bad profile token: $p" >&2; exit 1 ;; - esac - done - declare -f scenario_break scenario_fix scenario_assert >/dev/null \ - || { echo "missing scenario_break/fix/assert" >&2; exit 1; }' \ - _probe "$f" 2>&1 >/dev/null) \ - || fatal "scenario contract violation in $base: $probe_err" -} - _profile_matches() { # <profiles> -> 0 if scenario runs under $FS_PROFILE local p for p in $1; do @@ -283,44 +258,6 @@ import json; print(len(json.load(open('$RESULTS_DIR/bootstrap-status.json'))['me PASS=() FAIL=() -run_scenario() { # <index> - local i="$1" name="${S_NAMES[$1]}" f="${S_FILES[$1]}" - section "Scenario: $name — ${S_DESCS[$1]}" - # Sourced in the runner's shell so break/fix/assert see the helpers; - # unset afterwards so the next scenario can't inherit stale functions. - # shellcheck disable=SC1090 - source "$f" - local ok=true - step "break" - if ! scenario_break; then - error "$name: break step failed" - ok=false - fi - if [ "$ok" = "true" ]; then - step "fix" - if ! scenario_fix; then - error "$name: fix step failed" - ok=false - fi - fi - if [ "$ok" = "true" ]; then - step "assert" - if ! scenario_assert; then - error "$name: post-state assertion failed" - ok=false - fi - fi - unset -f scenario_break scenario_fix scenario_assert - unset SCENARIO_DESC SCENARIO_GROUP SCENARIO_PROFILES - if [ "$ok" = "true" ]; then - success "PASS: $name" - PASS+=("$name") - else - error "FAIL: $name" - FAIL+=("$name") - fi -} - section "Preparing VM" stop_qemu 2>/dev/null || true step "Restoring clean-install snapshot" diff --git a/scripts/testing/run-test-baremetal.sh b/scripts/testing/run-test-baremetal.sh index d837389..4692fda 100755 --- a/scripts/testing/run-test-baremetal.sh +++ b/scripts/testing/run-test-baremetal.sh @@ -23,6 +23,7 @@ source "$SCRIPT_DIR/lib/logging.sh" source "$SCRIPT_DIR/lib/validation.sh" # live helpers: ssh_cmd, capture_*_state, analyze_log_diff, generate_issue_report source "$SCRIPT_DIR/lib/vm-utils.sh" # inject_root_key + SSH_OPTS/SSH_KEY_OPT for key auth source "$SCRIPT_DIR/lib/testinfra.sh" # run_testinfra_validation (authoritative validator) +source "$SCRIPT_DIR/lib/run-common.sh" # Parse arguments ROLLBACK_FIRST=false @@ -203,30 +204,25 @@ if ! $VALIDATE_ONLY; then # Start archsetup in background ssh_cmd "cd /tmp/archsetup-test && nohup bash archsetup > $REMOTE_LOG 2>&1 &" - success "ArchSetup started in background" + sleep 3 + if archsetup_process_running ssh_cmd; then + success "ArchSetup started in background" + else + fatal "ArchSetup process not found after launch" + fi # Poll for completion step "Monitoring archsetup progress" - POLL_COUNT=0 MAX_POLLS=180 # 90 minutes max - while [ $POLL_COUNT -lt $MAX_POLLS ]; do - if ssh_cmd "ps aux | grep '[b]ash archsetup' > /dev/null" 2>/dev/null; then - sleep 30 - POLL_COUNT=$((POLL_COUNT + 1)) - if [ $((POLL_COUNT % 10)) -eq 0 ]; then - ELAPSED_MINS=$((POLL_COUNT / 2)) - info "Still running... ($ELAPSED_MINS minutes elapsed)" - # Show last line of progress - LAST_LINE=$(ssh_cmd "tail -1 $REMOTE_LOG 2>/dev/null" || echo "") - [ -n "$LAST_LINE" ] && info " $LAST_LINE" - fi - else - break - fi - done + baremetal_progress() { + local last_line + info "Still running... ($(("$1" / 2)) minutes elapsed)" + last_line=$(ssh_cmd "tail -1 $REMOTE_LOG 2>/dev/null" || echo "") + [ -n "$last_line" ] && info " $last_line" + } - if [ $POLL_COUNT -ge $MAX_POLLS ]; then + if ! wait_for_archsetup ssh_cmd "$MAX_POLLS" 30 10 baremetal_progress; then error "ArchSetup timed out after 90 minutes" ARCHSETUP_COMPLETED=timeout else @@ -294,31 +290,14 @@ fi section "Generating Test Report" REPORT_FILE="$TEST_RESULTS_DIR/test-report.txt" -cat > "$REPORT_FILE" << EOFREPORT -======================================== -Bare Metal ArchSetup Test Report -======================================== - -Test ID: $TIMESTAMP -Date: $(date +'%Y-%m-%d %H:%M:%S') -Target: $TARGET_HOST -Test Method: Bare Metal ZFS - -Results: - ArchSetup Completed: $ARCHSETUP_COMPLETED (completion marker, not the installer's exit code) - Validation: $(if $TEST_PASSED; then echo "PASSED"; else echo "FAILED"; fi) - -Validation Summary: - Passed: $VALIDATION_PASSED - Failed: $VALIDATION_FAILED - Warnings: $VALIDATION_WARNINGS - -Artifacts: - Log file: $LOGFILE - Report: $REPORT_FILE - Results: $TEST_RESULTS_DIR/ - -EOFREPORT +printf -v RUN_REPORT_ARTIFACTS \ + 'Artifacts:\n Log file: %s\n Report: %s\n Results: %s/' \ + "$LOGFILE" "$REPORT_FILE" "$TEST_RESULTS_DIR" +write_archsetup_test_report \ + "$REPORT_FILE" "Bare Metal ArchSetup Test Report" "$TIMESTAMP" \ + "Bare Metal ZFS" "Target: $TARGET_HOST" "$ARCHSETUP_COMPLETED" \ + "$TEST_PASSED" "$VALIDATION_PASSED" "$VALIDATION_FAILED" \ + "$VALIDATION_WARNINGS" "$RUN_REPORT_ARTIFACTS" info "Test report saved: $REPORT_FILE" diff --git a/scripts/testing/run-test.sh b/scripts/testing/run-test.sh index a5c3691..0f57678 100755 --- a/scripts/testing/run-test.sh +++ b/scripts/testing/run-test.sh @@ -25,6 +25,7 @@ source "$SCRIPT_DIR/lib/vm-utils.sh" source "$SCRIPT_DIR/lib/network-diagnostics.sh" source "$SCRIPT_DIR/lib/validation.sh" source "$SCRIPT_DIR/lib/testinfra.sh" +source "$SCRIPT_DIR/lib/run-common.sh" # Parse arguments KEEP_VM=false @@ -248,7 +249,11 @@ sshpass -p "$ROOT_PASSWORD" ssh -T -n $SSH_OPTS \ # Verify the process started sleep 3 -if vm_exec "$ROOT_PASSWORD" "pgrep -f 'bash archsetup'" >> "$LOGFILE" 2>/dev/null; then +vm_transport() { + vm_exec "$ROOT_PASSWORD" "$@" +} + +if archsetup_process_running vm_transport; then success "ArchSetup started in background on VM" else fatal "ArchSetup process not found after launch" @@ -256,29 +261,14 @@ fi # Poll for completion step "Monitoring archsetup progress (polling every 30 seconds)..." -POLL_COUNT=0 MAX_POLLS=300 # 150 minutes max (300 * 30 seconds); a full install with heavy # AUR builds (e.g. vagrant) can exceed 90 min on a slow mirror -while [ $POLL_COUNT -lt $MAX_POLLS ]; do - # Check if archsetup process is still running - if vm_exec "$ROOT_PASSWORD" "ps aux | grep '[b]ash archsetup' > /dev/null" 2>/dev/null; then - # Still running, wait and continue - sleep 30 - POLL_COUNT=$((POLL_COUNT + 1)) - - # Show progress every 5 minutes - if [ $((POLL_COUNT % 10)) -eq 0 ]; then - ELAPSED_MINS=$((POLL_COUNT / 2)) - info "Still running... ($ELAPSED_MINS minutes elapsed)" - fi - else - # Process finished - break - fi -done +vm_progress() { + info "Still running... ($(("$1" / 2)) minutes elapsed)" +} -if [ $POLL_COUNT -ge $MAX_POLLS ]; then +if ! wait_for_archsetup vm_transport "$MAX_POLLS" 30 10 vm_progress; then error "ArchSetup timed out after 150 minutes" ARCHSETUP_COMPLETED=timeout else @@ -357,44 +347,20 @@ fi section "Generating Test Report" REPORT_FILE="$TEST_RESULTS_DIR/test-report.txt" -cat > "$REPORT_FILE" << EOFREPORT -======================================== -ArchSetup Test Report -======================================== - -Test ID: $TIMESTAMP -Date: $(date +'%Y-%m-%d %H:%M:%S') -Test Method: QEMU snapshot-based - -VM Configuration: - Disk: $DISK_PATH - Snapshot: $SNAPSHOT_NAME - SSH: localhost:$SSH_PORT - -Results: - ArchSetup Completed: $ARCHSETUP_COMPLETED (completion marker, not the installer's exit code) - Validation: $(if $TEST_PASSED; then echo "PASSED"; else echo "FAILED"; fi) - -Validation Summary: - Passed: $VALIDATION_PASSED - Failed: $VALIDATION_FAILED - Warnings: $VALIDATION_WARNINGS - -Issue Attribution: - ArchSetup Issues: ${#ARCHSETUP_ISSUES[@]} - Base Install Issues: ${#BASE_INSTALL_ISSUES[@]} - Unknown Issues: ${#UNKNOWN_ISSUES[@]} - -Artifacts: - Log file: $LOGFILE - Report: $REPORT_FILE - Results: $TEST_RESULTS_DIR/ - Issue Report: $TEST_RESULTS_DIR/issue-report.txt - Pre-install logs: $TEST_RESULTS_DIR/pre-install/ - Post-install logs: $TEST_RESULTS_DIR/post-install/ - Analysis: $TEST_RESULTS_DIR/analysis/ - -EOFREPORT +printf -v RUN_REPORT_CONTEXT \ + 'VM Configuration:\n Disk: %s\n Snapshot: %s\n SSH: localhost:%s' \ + "$DISK_PATH" "$SNAPSHOT_NAME" "$SSH_PORT" +printf -v RUN_REPORT_ARTIFACTS \ + 'Issue Attribution:\n ArchSetup Issues: %s\n Base Install Issues: %s\n Unknown Issues: %s\n\nArtifacts:\n Log file: %s\n Report: %s\n Results: %s/\n Issue Report: %s/issue-report.txt\n Pre-install logs: %s/pre-install/\n Post-install logs: %s/post-install/\n Analysis: %s/analysis/' \ + "${#ARCHSETUP_ISSUES[@]}" "${#BASE_INSTALL_ISSUES[@]}" \ + "${#UNKNOWN_ISSUES[@]}" "$LOGFILE" "$REPORT_FILE" "$TEST_RESULTS_DIR" \ + "$TEST_RESULTS_DIR" "$TEST_RESULTS_DIR" "$TEST_RESULTS_DIR" \ + "$TEST_RESULTS_DIR" +write_archsetup_test_report \ + "$REPORT_FILE" "ArchSetup Test Report" "$TIMESTAMP" \ + "QEMU snapshot-based" "$RUN_REPORT_CONTEXT" "$ARCHSETUP_COMPLETED" \ + "$TEST_PASSED" "$VALIDATION_PASSED" "$VALIDATION_FAILED" \ + "$VALIDATION_WARNINGS" "$RUN_REPORT_ARTIFACTS" info "Test report saved: $REPORT_FILE" diff --git a/scripts/wipedisk b/scripts/wipedisk index b833407..9b1b4ab 100644 --- a/scripts/wipedisk +++ b/scripts/wipedisk @@ -1,9 +1,29 @@ #!/usr/bin/env bash # SPDX-License-Identifier: GPL-3.0-or-later # Craig Jennings <c@cjennings.net> -# identify disk and erase +# identify disk and erase + +# Overridable so the test suite can point at a fixture directory, the way +# nvidia_preflight_report takes NVIDIA_DRM_GLOB. +by_id_dir="${WIPEDISK_BY_ID:-/dev/disk/by-id}" + +# Whole disks only. /dev/disk/by-id publishes a -partN entry for every +# partition alongside the disks (18 entries on this machine, 12 of them +# partitions), and the prompt below promises a disk. Globbed rather than +# `ls | grep` so a name with whitespace can't split into two menu entries. +all_disk_ids=() +for entry in "$by_id_dir"/*; do + [ -e "$entry" ] || continue # unmatched glob stays literal + name="${entry##*/}" + [[ "$name" =~ -part[0-9]+$ ]] && continue + all_disk_ids+=("$name") +done + +if [ "${#all_disk_ids[@]}" -eq 0 ]; then + echo "No disks found in $by_id_dir." >&2 + exit 1 +fi -all_disk_ids=( $(ls /dev/disk/by-id/) ) echo ""; echo "Select the disk id to use. All data will be erased." select disk_id in "${all_disk_ids[@]}"; do # ensure valid selection @@ -16,16 +36,58 @@ select disk_id in "${all_disk_ids[@]}"; do done # Confirm the selected disk -read -p "Confirm: '$selection' [y/n]? " choice +read -r -p "Confirm: '$selection' [y/n]? " choice if [[ "$choice" != "y" ]]; then echo "Exiting..." exit 1 fi -DISK="/dev/disk/by-id/$selection" +DISK="$by_id_dir/$selection" echo ""; echo "### Erasing Disk" -blkdiscard -f "${DISK}" || true # discard all sectors on flash-based storage -sgdisk --zap-all "${DISK}" # clear the disk + +# blkdiscard opens the device O_EXCL by default (util-linux >= 2.36) and +# refuses a disk something is holding -- a mounted filesystem, an md member, an +# LVM PV. That refusal is this script's safety gate, and it has to run BEFORE +# anything destructive. Passing -f disables the exclusive open, which turned +# "refuse the wrong disk" into "discard a live filesystem, then let sgdisk fail +# and tell the user nothing happened". The gate is the kernel's answer, not a +# heuristic of ours, so no -f. +discarded=true +discard_err=$(blkdiscard "${DISK}" 2>&1) || discarded=false + +# A discard can fail for two very different reasons. "Operation not supported" +# means the hardware has no discard -- the run continues and the closing +# message says the data is still there. Anything else means the device was +# refused, and nothing has been written yet, so stop while that is still true. +if [ "$discarded" = false ] \ + && ! printf '%s' "$discard_err" | grep -qi 'not supported'; then + echo "" >&2 + echo "REFUSED: '$selection' is in use. Nothing was erased." >&2 + [ -n "$discard_err" ] && echo " $discard_err" >&2 + echo " Unmount its filesystems and stop any md/LVM/ZFS holder, then run" >&2 + echo " this again." >&2 + exit 1 +fi + +# sgdisk refuses a busy device too. Reaching here means blkdiscard already +# accepted the disk, so a failure now is something else -- report it rather +# than announcing an erase that did not happen. +if ! sgdisk --zap-all "${DISK}"; then + echo "" >&2 + echo "FAILED: could not clear the partition table on '$selection'." >&2 + echo " The sectors were discarded but the partition table was not" >&2 + echo " rewritten. Check the device and run this again." >&2 + exit 1 +fi echo "" -echo "Disk erased."
\ No newline at end of file +if [ "$discarded" = true ]; then + echo "Disk erased. Sectors discarded and the partition table cleared." +else + # Say what actually happened. sgdisk --zap-all destroys partition tables, + # not data, so without a successful discard every byte is still readable. + echo "Partition table cleared on '$selection'." + echo "NOTE: this device did not support discard, so the data is still" + echo " present and recoverable. For a disposal-grade wipe use the" + echo " drive's own secure erase (nvme format, hdparm) or overwrite it." +fi diff --git a/scripts/zfs-replicate b/scripts/zfs-replicate index 02ffcf5..a4cb15d 100755 --- a/scripts/zfs-replicate +++ b/scripts/zfs-replicate @@ -25,9 +25,14 @@ YELLOW='\033[1;33m' RED='\033[0;31m' NC='\033[0m' -info() { echo -e "${GREEN}[INFO]${NC} $1"; } -warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; } +# Diagnostics go to stderr, never stdout. determine_host below runs inside a +# command substitution, so anything these print on stdout would be captured +# into TRUENAS_HOST instead of reaching the terminal or the journal -- which is +# exactly how an unreachable TrueNAS used to abort this script with exit 1 and +# no output at all. stderr also keeps the captured hostname clean. +info() { echo -e "${GREEN}[INFO]${NC} $1" >&2; } +warn() { echo -e "${YELLOW}[WARN]${NC} $1" >&2; } +error() { echo -e "${RED}[ERROR]${NC} $1" >&2; exit 1; } command -v syncoid >/dev/null 2>&1 || error "syncoid not found. Install sanoid package." @@ -58,6 +63,12 @@ fi info "Starting ZFS replication to $TRUENAS_HOST" echo "" +# A failed dataset must not stop the others -- but it must not be forgotten +# either. This runs as a systemd oneshot on a nightly timer, so the exit code +# is the only signal anyone sees; exiting 0 after every dataset failed made a +# backup that had not run in months look identical to a working one. +failed=0 + for dataset in $DATASETS; do dest="$TRUENAS_USER@$TRUENAS_HOST:$TRUENAS_POOL/$BACKUP_PATH/${dataset#zroot/}" info "Replicating $dataset -> $dest" @@ -66,8 +77,14 @@ for dataset in $DATASETS; do info " Success" else warn " Failed (will retry next run)" + failed=$((failed + 1)) fi - echo "" + echo "" >&2 done +if [ "$failed" -gt 0 ]; then + warn "Replication complete with $failed of $(echo "$DATASETS" | wc -w) datasets failed." + exit 1 +fi + info "Replication complete." |