summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCraig Jennings <c@cjennings.net>2025-11-15 13:32:35 -0600
committerCraig Jennings <c@cjennings.net>2025-11-15 13:32:35 -0600
commit11479b3ba3ab9dafde70b2d51afeac9c47657ad5 (patch)
treefa8baf70af153bd2e3ea90a1f12ac5bc85835f4c
parentfbae1614bf8cff06366de69f5f1694d0c813824c (diff)
feat(tests): Improve test failure reporting in Makefile
Major improvements to test output clarity and failure diagnostics: Output Improvements: - Clean aligned output showing test file + pass/fail status - Show test count for passing files: "✓ (11 tests)" - Clear "✗ FAILED" marker for failures (no verbose noise) - Suppress passing test details (reduces output by ~95%) Failure Summary: - Clear visual separator box for failures - List all failed files with bullet points - Show specific failed test names with line numbers - Provide copy-paste commands to re-run failures Technical Changes: - Capture test output and only display on failure - Extract test count from ERT output for passing tests - Save failure details to temp files for summary - Apply same improvements to both unit and integration tests Benefits: - Instantly see which tests failed (no scrolling) - Know exactly which test names failed - Get line numbers for debugging - Copy-paste commands to re-run specific failures - Dramatically reduced output verbosity Example summary: Failed test files: • test-mousetrap-mode--get-profile-for-mode.el Errors: FAILED test-foo (line 15) Run individual failing tests with: make test-file FILE=test-mousetrap-mode--get-profile-for-mode.el 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
-rw-r--r--Makefile86
1 files changed, 77 insertions, 9 deletions
diff --git a/Makefile b/Makefile
index 9a011b34..1a1924e2 100644
--- a/Makefile
+++ b/Makefile
@@ -89,15 +89,49 @@ test-all:
test-unit:
@echo "[i] Running unit tests ($(words $(UNIT_TESTS)) files)..."
+ @echo ""
@failed=0; \
+ failed_files=""; \
for test in $(UNIT_TESTS); do \
- echo " Testing $$test..."; \
- $(EMACS_TEST) -l ert -l $$test --eval "(ert-run-tests-batch-and-exit '(not (tag :slow)))" || failed=$$((failed + 1)); \
+ test_name=$$(basename $$test); \
+ printf " Testing %-60s " "$$test_name..."; \
+ output=$$($(EMACS_TEST) -l ert -l $$test --eval "(ert-run-tests-batch-and-exit '(not (tag :slow)))" 2>&1); \
+ result=$$?; \
+ if [ $$result -eq 0 ]; then \
+ pass_count=$$(echo "$$output" | grep -oP "Ran \K\d+" | head -1); \
+ echo "✓ ($$pass_count tests)"; \
+ else \
+ echo "✗ FAILED"; \
+ failed=$$((failed + 1)); \
+ failed_files="$$failed_files$$test_name "; \
+ echo "$$output" | grep -E "FAILED|unexpected|Error" > /tmp/test-failure-$$test_name.log; \
+ fi; \
done; \
+ echo ""; \
if [ $$failed -eq 0 ]; then \
- echo "✓ All unit tests passed"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
+ echo "✓ ALL UNIT TESTS PASSED"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
else \
- echo "✗ $$failed unit test file(s) failed"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
+ echo "✗ FAILURES DETECTED: $$failed test file(s) failed"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
+ echo ""; \
+ echo "Failed test files:"; \
+ for file in $$failed_files; do \
+ echo " • $$file"; \
+ if [ -f /tmp/test-failure-$$file.log ]; then \
+ echo " Errors:"; \
+ sed 's/^/ /' /tmp/test-failure-$$file.log; \
+ rm /tmp/test-failure-$$file.log; \
+ fi; \
+ done; \
+ echo ""; \
+ echo "Run individual failing tests with:"; \
+ for file in $$failed_files; do \
+ echo " make test-file FILE=$$file"; \
+ done; \
+ echo ""; \
exit 1; \
fi
@@ -106,16 +140,50 @@ test-integration:
echo "No integration tests found"; \
exit 0; \
fi
- @echo "Running integration tests ($(words $(INTEGRATION_TESTS)) files)..."
+ @echo "[i] Running integration tests ($(words $(INTEGRATION_TESTS)) files)..."
+ @echo ""
@failed=0; \
+ failed_files=""; \
for test in $(INTEGRATION_TESTS); do \
- echo " Testing $$test..."; \
- $(EMACS_TEST) -l ert -l $$test --eval "(ert-run-tests-batch-and-exit '(not (tag :slow)))" || failed=$$((failed + 1)); \
+ test_name=$$(basename $$test); \
+ printf " Testing %-60s " "$$test_name..."; \
+ output=$$($(EMACS_TEST) -l ert -l $$test --eval "(ert-run-tests-batch-and-exit '(not (tag :slow)))" 2>&1); \
+ result=$$?; \
+ if [ $$result -eq 0 ]; then \
+ pass_count=$$(echo "$$output" | grep -oP "Ran \K\d+" | head -1); \
+ echo "✓ ($$pass_count tests)"; \
+ else \
+ echo "✗ FAILED"; \
+ failed=$$((failed + 1)); \
+ failed_files="$$failed_files$$test_name "; \
+ echo "$$output" | grep -E "FAILED|unexpected|Error" > /tmp/test-failure-$$test_name.log; \
+ fi; \
done; \
+ echo ""; \
if [ $$failed -eq 0 ]; then \
- echo "✓ All integration tests passed"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
+ echo "✓ ALL INTEGRATION TESTS PASSED"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
else \
- echo "✗ $$failed integration test file(s) failed"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
+ echo "✗ FAILURES DETECTED: $$failed test file(s) failed"; \
+ echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
+ echo ""; \
+ echo "Failed test files:"; \
+ for file in $$failed_files; do \
+ echo " • $$file"; \
+ if [ -f /tmp/test-failure-$$file.log ]; then \
+ echo " Errors:"; \
+ sed 's/^/ /' /tmp/test-failure-$$file.log; \
+ rm /tmp/test-failure-$$file.log; \
+ fi; \
+ done; \
+ echo ""; \
+ echo "Run individual failing tests with:"; \
+ for file in $$failed_files; do \
+ echo " make test-file FILE=$$file"; \
+ done; \
+ echo ""; \
exit 1; \
fi