diff options
| -rw-r--r-- | Makefile | 22 | ||||
| -rw-r--r-- | scripts/coverage-summary.el | 192 | ||||
| -rw-r--r-- | tests/Makefile | 3 | ||||
| -rw-r--r-- | tests/test-pearl-coverage-summary.el | 163 |
4 files changed, 378 insertions, 2 deletions
@@ -27,7 +27,7 @@ ALL_TESTS = $(filter-out $(TEST_DIR)/test-bootstrap.el, \ .PHONY: help test test-all test-unit test-integration test-file test-one test-name \ count list validate lint check-deps clean \ - setup compile coverage coverage-clean + setup compile coverage coverage-summary coverage-clean help: @$(MAKE) -C $(TEST_DIR) help @@ -148,6 +148,26 @@ coverage: coverage-clean $(COVERAGE_DIR) echo "[!] No coverage file produced; check that undercover is installed"; \ exit 1; \ fi + @# Print the human-readable summary after a local run. CI emits + @# coveralls.json (not simplecov.json) and the upload action reports + @# instead, so skip the terminal summary there. + @if [ -z "$$CI" ] && [ -f $(COVERAGE_FILE) ]; then \ + $(MAKE) --no-print-directory coverage-summary; \ + fi + +# Print a human-readable summary of the SimpleCov report: per-file +# covered/total lines + percent, a line-weighted project figure, and a +# source-weighted figure where a tracked source missing from the report +# counts as 0% (so an un-instrumented source shows up rather than vanishing +# from the numbers). Run on its own against an existing report, or +# automatically at the tail of 'make coverage'. +coverage-summary: + @if [ ! -f $(COVERAGE_FILE) ]; then \ + echo "[!] No coverage report at $(COVERAGE_FILE). Run 'make coverage' first."; \ + exit 1; \ + fi + @$(EMACS_BATCH) -L scripts -l coverage-summary \ + --eval '(pearl-coverage-print-summary "$(COVERAGE_FILE)" (list "$(SOURCE_FILE)") "$(CURDIR)")' coverage-clean: @rm -f $(COVERAGE_FILE) diff --git a/scripts/coverage-summary.el b/scripts/coverage-summary.el new file mode 100644 index 0000000..2be2bac --- /dev/null +++ b/scripts/coverage-summary.el @@ -0,0 +1,192 @@ +;;; coverage-summary.el --- Terminal coverage summary for pearl -*- lexical-binding: t; -*- + +;; Copyright (C) 2026 Craig Jennings + +;; Author: Craig Jennings <c@cjennings.net> + +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Batch helper behind `make coverage-summary'. Parses the SimpleCov JSON +;; that `make coverage' writes and prints a terminal summary instead of just +;; the report's file size: +;; +;; - per tracked source file: covered/total executable lines and a percent, +;; worst-covered first; +;; - a line-weighted project figure over files present in the report; +;; - a source-weighted figure where a tracked source missing from the +;; report counts as 0% rather than being silently dropped; and +;; - an explicit list of tracked sources absent from the report. +;; +;; The missing-source handling is the robustness win: a source that never got +;; instrumented shows as 0% instead of vanishing from the numbers. +;; +;; Adapted from the multi-module version in the author's Emacs config; pearl +;; ships a single source file today, so this is generalized over a list of +;; tracked sources and stays self-contained (no dependency on the editor's +;; coverage engine). + +;;; Code: + +(require 'json) +(require 'seq) +(require 'subr-x) + +(defun pearl-coverage--read-json (file) + "Parse FILE as SimpleCov JSON and return the decoded hash table. +Signal `user-error' when FILE is missing or the JSON is malformed." + (unless (file-exists-p file) + (user-error "Coverage report not found: %s (run 'make coverage' first)" file)) + (let ((json-object-type 'hash-table) + (json-array-type 'list) + (json-key-type 'string)) + (condition-case err + (json-read-file file) + (error (user-error "Malformed coverage JSON in %s: %s" + file (error-message-string err)))))) + +(defun pearl-coverage--collect (file predicate) + "Return path -> (line -> t) hash for lines in FILE matching PREDICATE. +PREDICATE receives each line's SimpleCov hits entry (a number, or nil for a +non-executable line). Coverage is unioned across every top-level test-name +key so undercover's `:merge-report' output accumulates correctly. A path is +recorded even when no line matches, so callers can tell \"present but empty\" +from \"absent\"." + (let ((data (pearl-coverage--read-json file)) + (result (make-hash-table :test 'equal))) + (maphash + (lambda (_test-name section) + (when (hash-table-p section) + (let ((coverage (gethash "coverage" section))) + (when (hash-table-p coverage) + (maphash + (lambda (path hits-list) + (let ((lines (or (gethash path result) + (make-hash-table :test 'eql))) + (line-num 1)) + (dolist (hits hits-list) + (when (funcall predicate hits) + (puthash line-num t lines)) + (setq line-num (1+ line-num))) + (puthash path lines result))) + coverage))))) + data) + result)) + +(defun pearl-coverage--parse-simplecov (file) + "Return path -> (line -> t) hash for HIT lines (hits > 0) in FILE." + (pearl-coverage--collect file (lambda (h) (and (numberp h) (> h 0))))) + +(defun pearl-coverage--executable-lines (file) + "Return path -> (line -> t) hash for EXECUTABLE lines in FILE. +Executable means the SimpleCov entry is a number (hit or unhit); null entries +(blank lines, comments) are excluded." + (pearl-coverage--collect file #'numberp)) + +(defun pearl-coverage--line-count (table path) + "Return the number of recorded lines for PATH in TABLE, or 0 when absent." + (let ((lines (gethash path table))) + (if (hash-table-p lines) (hash-table-count lines) 0))) + +(defun pearl-coverage--find-key (table abs) + "Return the key in TABLE matching ABS, or nil. +Matches by expanded path first, then by basename, so a report whose paths are +written in a slightly different but equivalent form still resolves." + (let ((want-base (file-name-nondirectory abs)) + (found nil)) + (catch 'done + (maphash (lambda (k _v) + (when (or (string= (expand-file-name k) abs) + (string= (file-name-nondirectory k) want-base)) + (setq found k) + (throw 'done k))) + table)) + found)) + +(defun pearl-coverage--records (report-file source-files project-root) + "Return per-file coverage records for SOURCE-FILES from REPORT-FILE. +SOURCE-FILES are paths relative to PROJECT-ROOT. Each record is a plist: + + (:path REL :covered N :total N :present BOOL :percent FLOAT) + +A source absent from the report has :present nil, :covered 0, :total 0, and +:percent 0.0 — it counts as 0%, not 100%. A present source with no executable +lines is 100% (nothing left to cover)." + (let ((covered (pearl-coverage--parse-simplecov report-file)) + (executable (pearl-coverage--executable-lines report-file)) + (root (file-name-as-directory (expand-file-name project-root)))) + (mapcar + (lambda (rel) + (let* ((abs (expand-file-name rel root)) + (key (pearl-coverage--find-key executable abs)) + (present (and key t)) + (total (if key (pearl-coverage--line-count executable key) 0)) + (cov (if key (pearl-coverage--line-count covered key) 0)) + (percent (cond ((not present) 0.0) + ((zerop total) 100.0) + (t (/ (* 100.0 cov) total))))) + (list :path rel :covered cov :total total + :present present :percent percent))) + source-files))) + +(defun pearl-coverage-summary-text (report-file source-files project-root) + "Return a coverage summary string for SOURCE-FILES from REPORT-FILE. +SOURCE-FILES are paths relative to PROJECT-ROOT." + (let* ((records (pearl-coverage--records report-file source-files project-root)) + (present (seq-filter (lambda (r) (plist-get r :present)) records)) + (missing (seq-remove (lambda (r) (plist-get r :present)) records)) + (total-cov (apply #'+ (mapcar (lambda (r) (plist-get r :covered)) present))) + (total-lines (apply #'+ (mapcar (lambda (r) (plist-get r :total)) present))) + (line-pct (if (> total-lines 0) (/ (* 100.0 total-cov) total-lines) 100.0)) + (file-score (apply #'+ (mapcar (lambda (r) (plist-get r :percent)) records))) + (file-pct (if records (/ file-score (length records)) 0.0)) + (sorted (sort (copy-sequence records) + (lambda (a b) (< (plist-get a :percent) (plist-get b :percent))))) + (width (apply #'max 8 (mapcar (lambda (r) (length (plist-get r :path))) records))) + (row-format (format " %%-%ds %%5d/%%-5d (%%5.1f%%%%)\n" width)) + (miss-format (format " %%-%ds not in report (counts as 0%%%%)\n" width))) + (with-temp-buffer + (insert "Coverage Summary\n================\n\n") + (insert "Per-file coverage (worst first):\n") + (dolist (r sorted) + (if (plist-get r :present) + (insert (format row-format + (plist-get r :path) + (plist-get r :covered) + (plist-get r :total) + (plist-get r :percent))) + (insert (format miss-format (plist-get r :path))))) + (insert "\n") + (insert (format "Line coverage: %d/%d lines (%.1f%%) across %d file%s in report\n" + total-cov total-lines line-pct (length present) + (if (= 1 (length present)) "" "s"))) + (insert (format "Source coverage: %.1f%% (%d tracked, %d missing; missing counts as 0%%)\n" + file-pct (length present) (length missing))) + (when missing + (insert (format "\nNot in coverage report: %d source%s\n" + (length missing) (if (= 1 (length missing)) "" "s"))) + (dolist (r missing) + (insert (format " %s\n" (plist-get r :path))))) + (buffer-string)))) + +(defun pearl-coverage-print-summary (report-file source-files project-root) + "Print the coverage summary for SOURCE-FILES to standard output. +SOURCE-FILES are paths relative to PROJECT-ROOT. Entry point for +`make coverage-summary'." + (princ "\n") + (princ (pearl-coverage-summary-text report-file source-files project-root))) + +(provide 'pearl-coverage-summary) +;;; coverage-summary.el ends here diff --git a/tests/Makefile b/tests/Makefile index c011359..341bfdb 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -293,7 +293,8 @@ help: @echo "Project-root targets (run from project root):" @echo " make setup - Install all deps via eask" @echo " make compile - Byte-compile pearl.el" - @echo " make coverage - Generate simplecov JSON via undercover" + @echo " make coverage - Generate simplecov JSON via undercover (+ summary)" + @echo " make coverage-summary - Print covered/total + percent from the last report" @echo "" @echo "Tagging tests as :slow:" @echo " (ert-deftest test-foo () :tags '(:slow) ...) — excluded by default" diff --git a/tests/test-pearl-coverage-summary.el b/tests/test-pearl-coverage-summary.el new file mode 100644 index 0000000..79de30a --- /dev/null +++ b/tests/test-pearl-coverage-summary.el @@ -0,0 +1,163 @@ +;;; test-pearl-coverage-summary.el --- Tests for the coverage summary script -*- lexical-binding: t; -*- + +;; Copyright (C) 2026 Craig Jennings + +;; Author: Craig Jennings <c@cjennings.net> + +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Unit tests for scripts/coverage-summary.el — the batch helper behind +;; `make coverage-summary'. Drives the SimpleCov parsers, the per-file +;; record builder, and the formatted text on synthetic reports written to +;; temp files (no committed fixtures). Covers the normal percentage path, +;; boundary cases (fully covered, no executable lines, multi-key union), and +;; error cases (missing file, malformed JSON, a tracked source absent from the +;; report counting as 0%). + +;;; Code: + +(require 'ert) +(load (expand-file-name "../scripts/coverage-summary.el") nil t) + +(defconst test-pearl-cov-root "/tmp/pearl-cov-test/" + "Synthetic project root used to anchor source-file paths in fixtures.") + +(defun test-pearl-cov--write (json) + "Write JSON to a temp file and return its path." + (let ((f (make-temp-file "pearl-cov" nil ".json"))) + (with-temp-file f (insert json)) + f)) + +(defun test-pearl-cov--report (hits-array) + "Write a SimpleCov report mapping pearl.el under the test root to HITS-ARRAY. +HITS-ARRAY is a JSON array literal string such as \"[null, 1, 0]\"." + (test-pearl-cov--write + (format "{\"undercover\": {\"coverage\": {\"%spearl.el\": %s}}}" + test-pearl-cov-root hits-array))) + +(defmacro test-pearl-cov--with (json-or-path varname &rest body) + "Bind VARNAME to a report path from JSON-OR-PATH, run BODY, then delete it." + (declare (indent 2)) + `(let ((,varname ,json-or-path)) + (unwind-protect (progn ,@body) + (when (file-exists-p ,varname) (delete-file ,varname))))) + +;;; --- Parsers --- + +(ert-deftest test-pearl-coverage-parse-counts-only-hit-lines () + "`parse-simplecov' records only lines whose hit count is greater than zero." + (test-pearl-cov--with (test-pearl-cov--report "[null, 1, 0, 3, null, 0]") f + (let ((tbl (pearl-coverage--parse-simplecov f)) + (key (expand-file-name "pearl.el" test-pearl-cov-root))) + ;; lines 2 and 4 are hits; 3 and 6 are executable-but-unhit; 1,5 null + (should (= 2 (pearl-coverage--line-count tbl key)))))) + +(ert-deftest test-pearl-coverage-executable-counts-all-numeric-lines () + "`executable-lines' records every numeric entry, hit or not." + (test-pearl-cov--with (test-pearl-cov--report "[null, 1, 0, 3, null, 0]") f + (let ((tbl (pearl-coverage--executable-lines f)) + (key (expand-file-name "pearl.el" test-pearl-cov-root))) + ;; lines 2,3,4,6 are numeric; 1 and 5 are null + (should (= 4 (pearl-coverage--line-count tbl key)))))) + +(ert-deftest test-pearl-coverage-parse-unions-across-test-keys () + "Coverage under multiple top-level keys is unioned (undercover :merge-report)." + (test-pearl-cov--with + (test-pearl-cov--write + (format (concat "{\"a\": {\"coverage\": {\"%spearl.el\": [1, 0, null]}}," + " \"b\": {\"coverage\": {\"%spearl.el\": [0, 1, null]}}}") + test-pearl-cov-root test-pearl-cov-root)) + f + (let ((tbl (pearl-coverage--parse-simplecov f)) + (key (expand-file-name "pearl.el" test-pearl-cov-root))) + ;; line 1 hit in a, line 2 hit in b -> union is 2 covered lines + (should (= 2 (pearl-coverage--line-count tbl key)))))) + +;;; --- Records --- + +(ert-deftest test-pearl-coverage-record-normal-percentage () + "A half-covered file reports covered/total and a 50% figure." + (test-pearl-cov--with (test-pearl-cov--report "[null, 1, 0, 3, null, 0]") f + (let ((r (car (pearl-coverage--records f '("pearl.el") test-pearl-cov-root)))) + (should (plist-get r :present)) + (should (= 2 (plist-get r :covered))) + (should (= 4 (plist-get r :total))) + (should (< 49.9 (plist-get r :percent) 50.1))))) + +(ert-deftest test-pearl-coverage-record-fully-covered-is-100 () + "Every executable line hit reports 100%." + (test-pearl-cov--with (test-pearl-cov--report "[1, 2, 3]") f + (let ((r (car (pearl-coverage--records f '("pearl.el") test-pearl-cov-root)))) + (should (= 3 (plist-get r :covered))) + (should (= 3 (plist-get r :total))) + (should (< 99.9 (plist-get r :percent) 100.1))))) + +(ert-deftest test-pearl-coverage-record-no-executable-lines-is-100 () + "A present file with no executable lines is 100% — nothing left uncovered." + (test-pearl-cov--with (test-pearl-cov--report "[null, null]") f + (let ((r (car (pearl-coverage--records f '("pearl.el") test-pearl-cov-root)))) + (should (plist-get r :present)) + (should (= 0 (plist-get r :total))) + (should (< 99.9 (plist-get r :percent) 100.1))))) + +(ert-deftest test-pearl-coverage-record-missing-source-counts-as-zero () + "A tracked source absent from the report is flagged missing and counts as 0%." + (test-pearl-cov--with + (test-pearl-cov--write + (format "{\"u\": {\"coverage\": {\"%sother.el\": [1, 2]}}}" + test-pearl-cov-root)) + f + (let ((r (car (pearl-coverage--records f '("pearl.el") test-pearl-cov-root)))) + (should-not (plist-get r :present)) + (should (= 0 (plist-get r :covered))) + (should (= 0.0 (plist-get r :percent)))))) + +;;; --- Error handling --- + +(ert-deftest test-pearl-coverage-missing-report-errors () + "Parsing a nonexistent report signals a user-error." + (should-error (pearl-coverage--parse-simplecov "/nonexistent/pearl-cov.json") + :type 'user-error)) + +(ert-deftest test-pearl-coverage-malformed-json-errors () + "Malformed JSON signals a user-error rather than a raw json error." + (test-pearl-cov--with (test-pearl-cov--write "{ this is not json") f + (should-error (pearl-coverage--parse-simplecov f) :type 'user-error))) + +;;; --- Formatted text --- + +(ert-deftest test-pearl-coverage-summary-text-reports-both-numbers () + "The summary names the file, a line-coverage figure, and source coverage." + (test-pearl-cov--with (test-pearl-cov--report "[null, 1, 0, 3, null, 0]") f + (let ((txt (pearl-coverage-summary-text f '("pearl.el") test-pearl-cov-root))) + (should (string-match-p "pearl\\.el" txt)) + (should (string-match-p "Line coverage" txt)) + (should (string-match-p "Source coverage" txt)) + (should (string-match-p "50\\.0%" txt))))) + +(ert-deftest test-pearl-coverage-summary-text-lists-missing-source () + "The summary's missing section names a tracked source absent from the report." + (test-pearl-cov--with + (test-pearl-cov--write + (format "{\"u\": {\"coverage\": {\"%sother.el\": [1, 2]}}}" + test-pearl-cov-root)) + f + (let ((txt (pearl-coverage-summary-text f '("pearl.el") test-pearl-cov-root))) + (should (string-match-p "Not in coverage report" txt)) + (should (string-match-p "pearl\\.el" txt))))) + +(provide 'test-pearl-coverage-summary) +;;; test-pearl-coverage-summary.el ends here |