From 6b2fd20dfc6e2cba418d27c0955908db8ec19039 Mon Sep 17 00:00:00 2001 From: Craig Jennings Date: Sun, 21 Jun 2026 02:00:20 -0400 Subject: test: add make coverage-summary with untested-module detection A module no test loads never appears in undercover's report, so a line-weighted total silently skips it. The coverage-summary target counts such a file as 0% and weights the project number by file, so untested modules stay visible. I replaced scripts/coverage-summary.py, which only summarized files already in the report. make coverage now chains the summary, and CI prints it in the coverage step instead of a separate python call. The helper runs on stock Emacs (built-in json and seq), so it needs no dev deps. It lives in tracked scripts/ so CI can reach it. --- .github/workflows/ci.yml | 3 - Makefile | 18 ++++- scripts/coverage-summary.el | 173 ++++++++++++++++++++++++++++++++++++++++++++ scripts/coverage-summary.py | 56 -------------- 4 files changed, 190 insertions(+), 60 deletions(-) create mode 100644 scripts/coverage-summary.el delete mode 100755 scripts/coverage-summary.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8fa309..b963389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,9 +115,6 @@ jobs: - name: Run coverage run: make coverage - - name: Print coverage summary - run: python3 scripts/coverage-summary.py - - name: Upload coverage report uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index f80a494..88153c1 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,10 @@ TEST_UTIL_FILES = $(wildcard $(TEST_DIR)/testutil-*.el) # Coverage configuration COVERAGE_DIR = .coverage COVERAGE_FILE = $(COVERAGE_DIR)/simplecov.json +# Whole-project summary: prints per-file coverage plus source files absent from +# the report (modules no test loaded) counted as 0%. Self-contained, stock Emacs. +COVERAGE_SUMMARY = scripts/coverage-summary.el +COVERAGE_SOURCE_DIR = . # Plain emacs invocation (no package archives, used for parens-check) EMACS_BATCH = $(EMACS) --batch --no-site-file --no-site-lisp @@ -55,7 +59,7 @@ EASK_EMACS = $(EMACS_ENV) $(EASK) emacs --batch -q -L $(PROJECT_ROOT) -L $(TEST_ .PHONY: help test test-all test-smoke test-unit test-integration test-file test-name \ deps install-deps validate-parens validate compile lint \ - coverage coverage-clean \ + coverage coverage-summary coverage-clean \ clean clean-compiled clean-tests # Default target @@ -74,6 +78,7 @@ help: @echo "" @echo " Coverage:" @echo " make coverage - Generate simplecov JSON at $(COVERAGE_FILE)" + @echo " make coverage-summary - Print per-file + whole-project summary (untested modules at 0%)" @echo " make coverage-clean - Delete the coverage report file" @echo "" @echo " Validation:" @@ -231,6 +236,17 @@ coverage: coverage-clean $(COVERAGE_DIR) echo "[!] No coverage file produced; check that undercover is installed"; \ exit 1; \ fi + @$(MAKE) coverage-summary + +# Whole-project summary. Stock Emacs only (no Eask deps) — the script needs just +# the built-in `json' and `seq'. Counts a source file with no report entry as 0%. +coverage-summary: + @if [ ! -f $(COVERAGE_FILE) ]; then \ + echo "[!] No coverage report at $(COVERAGE_FILE). Run 'make coverage' first."; \ + exit 1; \ + fi + @$(EMACS) --batch -q -l $(COVERAGE_SUMMARY) \ + --eval '(cj/coverage-print-module-summary "$(COVERAGE_FILE)" "$(COVERAGE_SOURCE_DIR)" "$(CURDIR)")' coverage-clean: @rm -f $(COVERAGE_FILE) diff --git a/scripts/coverage-summary.el b/scripts/coverage-summary.el new file mode 100644 index 0000000..d40e273 --- /dev/null +++ b/scripts/coverage-summary.el @@ -0,0 +1,173 @@ +;;; coverage-summary.el --- Whole-project coverage summary from a SimpleCov report -*- lexical-binding: t; -*- + +;;; Commentary: +;; Batch helper for `make coverage-summary'. After `make coverage' writes an +;; undercover SimpleCov JSON report, this prints a per-file table, a project +;; number, and the source files present on disk but absent from the report. +;; +;; The value here is the missing-file detection: a module no test imports never +;; appears in the SimpleCov output, so it silently fails to drag the number +;; down. This script counts such a file as 0% and weights the project number +;; by file rather than by line, so untested modules are visible. +;; +;; Self-contained on purpose — lives in the project's tracked =scripts/= (so CI +;; can reach it) and must run with nothing but stock Emacs (`json' is built in). +;; The SimpleCov +;; JSON shape it parses is: +;; { : { "coverage": { : [null | 0 | int, ...] } } } +;; where a null entry is a non-executable line, 0 is executable-but-unhit, and +;; any positive integer is a hit. Data unions across multiple suite keys. +;; +;; CLI contract (mirrors the dotemacs original): +;; emacs --batch -l coverage-summary.el \ +;; --eval '(cj/coverage-print-module-summary REPORT SRC-DIR PROJECT-ROOT)' + +;;; Code: + +(require 'json) +(require 'seq) + +(defun cj/coverage-summary--parse-file (report-file) + "Parse REPORT-FILE (SimpleCov JSON) into per-file (COVERED . TOTAL) counts. + +Keys are absolute source-file paths. TOTAL counts executable lines (numeric +entries); COVERED counts hit lines (entries greater than zero). Data unions +across every top-level suite key. Signals `user-error' when REPORT-FILE is +missing or malformed." + (unless (file-exists-p report-file) + (user-error "Coverage report not found: %s" report-file)) + (let* ((json-object-type 'hash-table) + (json-array-type 'list) + (json-key-type 'string) + (data (condition-case err + (json-read-file report-file) + (error (user-error "Malformed coverage JSON in %s: %s" + report-file (error-message-string err))))) + ;; path -> (covered-set . total-set), line numbers held in hash sets so + ;; unioning across suites never double-counts a shared line. + (acc (make-hash-table :test 'equal))) + (maphash + (lambda (_suite section) + (when (hash-table-p section) + (let ((coverage (gethash "coverage" section))) + (when (hash-table-p coverage) + (maphash + (lambda (path hits-list) + (let* ((cell (or (gethash path acc) + (puthash path + (cons (make-hash-table :test 'eql) + (make-hash-table :test 'eql)) + acc))) + (covered (car cell)) + (total (cdr cell)) + (line 1)) + (dolist (hits hits-list) + (when (numberp hits) + (puthash line t total) + (when (> hits 0) (puthash line t covered))) + (setq line (1+ line))))) + coverage))))) + data) + (let ((result (make-hash-table :test 'equal))) + (maphash (lambda (path cell) + (puthash path + (cons (hash-table-count (car cell)) + (hash-table-count (cdr cell))) + result)) + acc) + result))) + +(defun cj/coverage-summary--under-dir (table source-dir project-root) + "Filter TABLE to files under SOURCE-DIR, re-keyed relative to PROJECT-ROOT." + (let ((result (make-hash-table :test 'equal)) + (source-dir (file-name-as-directory (expand-file-name source-dir))) + (project-root (file-name-as-directory (expand-file-name project-root)))) + (maphash + (lambda (path counts) + (let ((abs (expand-file-name path))) + (when (string-prefix-p source-dir abs) + (puthash (file-relative-name abs project-root) counts result)))) + table) + result)) + +(defun cj/coverage-summary--source-files (source-dir project-root) + "Return *.el files directly under SOURCE-DIR, relative to PROJECT-ROOT. +Sorted; compiled files and subdirectories are out of scope." + (let ((source-dir (file-name-as-directory (expand-file-name source-dir))) + (project-root (file-name-as-directory (expand-file-name project-root)))) + (sort (mapcar (lambda (p) (file-relative-name p project-root)) + (directory-files source-dir t "\\.el\\'")) + #'string<))) + +(defun cj/coverage-summary--missing (tracked source-dir project-root) + "Return source files present on disk but absent from TRACKED. +TRACKED is a list of project-relative paths (the report's keys under +SOURCE-DIR). The difference is the set of files no test exercised." + (seq-difference + (cj/coverage-summary--source-files source-dir project-root) + tracked + #'string=)) + +(defun cj/coverage-summary--file-pct (covered total) + "Return COVERED/TOTAL as a percentage. +A file with no executable lines (TOTAL 0) is 100% — nothing left uncovered." + (if (> total 0) (/ (* 100.0 covered) total) 100.0)) + +(defun cj/coverage-summary--project-pct (report-file source-dir project-root) + "Return the unit-weighted project coverage percentage. +Every tracked file contributes its own percentage; every source file missing +from REPORT-FILE contributes 0%. The result is the mean over all files under +SOURCE-DIR, so an untested module drags the number down instead of vanishing." + (let* ((tracked (cj/coverage-summary--under-dir + (cj/coverage-summary--parse-file report-file) + source-dir project-root)) + (keys (let (ks) (maphash (lambda (k _v) (push k ks)) tracked) ks)) + (missing (cj/coverage-summary--missing keys source-dir project-root)) + (score 0.0) + (total-count (+ (hash-table-count tracked) (length missing)))) + (maphash (lambda (_k counts) + (setq score (+ score (cj/coverage-summary--file-pct + (car counts) (cdr counts))))) + tracked) + (if (> total-count 0) (/ score total-count) 0.0))) + +(defun cj/coverage-summary-text (report-file source-dir project-root) + "Return a whole-project coverage summary for SOURCE-DIR from REPORT-FILE." + (let* ((tracked (cj/coverage-summary--under-dir + (cj/coverage-summary--parse-file report-file) + source-dir project-root)) + (rel-src (file-relative-name + (expand-file-name source-dir) + (file-name-as-directory (expand-file-name project-root)))) + (keys (let (ks) (maphash (lambda (k _v) (push k ks)) tracked) ks)) + (missing (cj/coverage-summary--missing keys source-dir project-root)) + (pct (cj/coverage-summary--project-pct report-file source-dir project-root))) + (with-temp-buffer + (insert (format "Coverage summary for %s\n\n" rel-src)) + (dolist (path (sort keys #'string<)) + (let* ((counts (gethash path tracked)) + (covered (car counts)) + (total (cdr counts))) + (insert (format " %6.1f%% %s (%d/%d lines)\n" + (cj/coverage-summary--file-pct covered total) + path covered total)))) + (insert (format "\nProject coverage: %.1f%% (%d tracked, %d missing, %d total; missing files count as 0%%)\n" + pct (hash-table-count tracked) (length missing) + (+ (hash-table-count tracked) (length missing)))) + (insert (format "\nNot in coverage report: %d file%s\n" + (length missing) (if (= 1 (length missing)) "" "s"))) + (if missing + (progn + (insert "These files had no coverage entry; they count as 0% in project coverage.\n") + (dolist (path (sort missing #'string<)) + (insert (format " %s\n" path)))) + (insert "Every source file appears in the coverage report.\n")) + (buffer-string)))) + +(defun cj/coverage-print-module-summary (report-file source-dir project-root) + "Print a whole-project coverage summary for SOURCE-DIR from REPORT-FILE." + (princ "\n") + (princ (cj/coverage-summary-text report-file source-dir project-root))) + +(provide 'coverage-summary) +;;; coverage-summary.el ends here diff --git a/scripts/coverage-summary.py b/scripts/coverage-summary.py deleted file mode 100755 index 9b7bc99..0000000 --- a/scripts/coverage-summary.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -"""Print a per-file and overall coverage summary from undercover's simplecov JSON. - -Usage: - python3 scripts/coverage-summary.py [path] - -If `path` is omitted, defaults to `.coverage/simplecov.json`. -Exit code is 0 on success, 1 if the JSON is missing or malformed. -""" - -import json -import os -import sys - - -def main(path: str) -> int: - try: - with open(path) as f: - data = json.load(f) - except FileNotFoundError: - print(f"error: {path} not found; run `make coverage` first", file=sys.stderr) - return 1 - except json.JSONDecodeError as exc: - print(f"error: {path} is not valid JSON: {exc}", file=sys.stderr) - return 1 - - try: - suite = data["undercover.el"]["coverage"] - except (KeyError, TypeError): - print(f"error: {path} does not look like an undercover simplecov report", - file=sys.stderr) - return 1 - - print(f'{"File":<30} {"Lines":>7} {"Covered":>8} {"Coverage":>10}') - print("-" * 60) - - total_lines = 0 - total_covered = 0 - for fname, lines in suite.items(): - relevant = [l for l in lines if l is not None] - covered = sum(1 for l in relevant if l > 0) - pct = 100.0 * covered / len(relevant) if relevant else 0.0 - total_lines += len(relevant) - total_covered += covered - short = os.path.basename(fname) - print(f"{short:<30} {len(relevant):>7} {covered:>8} {pct:>9.2f}%") - - print("-" * 60) - overall = 100.0 * total_covered / total_lines if total_lines else 0.0 - print(f'{"TOTAL":<30} {total_lines:>7} {total_covered:>8} {overall:>9.2f}%') - return 0 - - -if __name__ == "__main__": - target = sys.argv[1] if len(sys.argv) > 1 else ".coverage/simplecov.json" - sys.exit(main(target)) -- cgit v1.2.3