From 58434f406068887291342dece24a55b0887dd86b Mon Sep 17 00:00:00 2001 From: Craig Jennings Date: Sun, 31 May 2026 11:04:15 -0500 Subject: feat(scripts): add workflow-integrity checker + tests Startup's drift check catches index-vs-directory mismatches. This goes deeper: scripts/workflow-integrity.py runs six checks over the canonical .ai/workflows/: each file is indexed-or-a-plugin-of-an-indexed-engine, each index entry resolves to a file, each .ai/scripts/ reference resolves, each plugin maps to an indexed parent, each non-plugin workflow has an orientation section, and no trigger phrase is claimed by two workflows. Exit 1 on any finding. scripts/tests/workflow-integrity.bats covers the clean canonical state plus a fixture per breakage class. make test already globs scripts/tests/*.bats, so it's wired in. I calibrated against the 38 current workflows (clean). The orientation check accepts the real heading variety (Overview / Purpose / When to Use|Run / Status) and exempts plugins. --- scripts/tests/workflow-integrity.bats | 106 +++++++++++++++++++++++ scripts/workflow-integrity.py | 158 ++++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 scripts/tests/workflow-integrity.bats create mode 100755 scripts/workflow-integrity.py (limited to 'scripts') diff --git a/scripts/tests/workflow-integrity.bats b/scripts/tests/workflow-integrity.bats new file mode 100644 index 0000000..9152f89 --- /dev/null +++ b/scripts/tests/workflow-integrity.bats @@ -0,0 +1,106 @@ +#!/usr/bin/env bats +# Tests for scripts/workflow-integrity.py: clean on the real canonical +# workflows, and exit 1 with the right finding for each breakage class. + +setup() { + CHECKER="$(cd "$(dirname "$BATS_TEST_FILENAME")/../.." && pwd)/scripts/workflow-integrity.py" + TMP="$(mktemp -d)" + W="$TMP/.ai/workflows" + S="$TMP/.ai/scripts" + mkdir -p "$W" "$S" + touch "$S/helper.sh" + cat > "$W/INDEX.org" <<'EOF' +* Catalog +- =alpha.org= — the alpha workflow. + - Triggers: "do alpha" +- =engine.org= — an engine with plugins. + - Triggers: "run engine" +EOF + cat > "$W/alpha.org" <<'EOF' +* Overview +Alpha workflow. Uses .ai/scripts/helper.sh. +EOF + cat > "$W/engine.org" <<'EOF' +* Overview +The engine. +EOF + cat > "$W/engine.foo.org" <<'EOF' +* Adapter +The foo plugin of engine. +EOF +} + +teardown() { + rm -rf "$TMP" +} + +@test "workflow-integrity: real canonical workflows are clean" { + run python3 "$CHECKER" + [ "$status" -eq 0 ] + [[ "$output" == *"OK"* ]] +} + +@test "workflow-integrity: a valid fixture passes" { + run python3 "$CHECKER" "$W" + [ "$status" -eq 0 ] +} + +@test "workflow-integrity: an un-indexed non-plugin file is an orphan" { + cat > "$W/stray.org" <<'EOF' +* Overview +Not indexed, not a plugin. +EOF + run python3 "$CHECKER" "$W" + [ "$status" -eq 1 ] + [[ "$output" == *"orphan"* ]] + [[ "$output" == *"stray.org"* ]] +} + +@test "workflow-integrity: an index entry with no file is stale" { + echo '- =ghost.org= — a workflow that was deleted.' >> "$W/INDEX.org" + run python3 "$CHECKER" "$W" + [ "$status" -eq 1 ] + [[ "$output" == *"stale-index"* ]] + [[ "$output" == *"ghost.org"* ]] +} + +@test "workflow-integrity: a reference to a missing script fails" { + cat >> "$W/alpha.org" <<'EOF' +Also calls .ai/scripts/missing-tool.py which is gone. +EOF + run python3 "$CHECKER" "$W" + [ "$status" -eq 1 ] + [[ "$output" == *"bad-ref"* ]] + [[ "$output" == *"missing-tool.py"* ]] +} + +@test "workflow-integrity: a plugin whose engine is not indexed is flagged" { + cat > "$W/lonely.plugin.org" <<'EOF' +* Adapter +A plugin with no indexed parent engine. +EOF + run python3 "$CHECKER" "$W" + [ "$status" -eq 1 ] + [[ "$output" == *"orphan-plugin"* ]] +} + +@test "workflow-integrity: an indexed workflow with no orientation section is flagged" { + echo '- =bare.org= — a bare workflow.' >> "$W/INDEX.org" + cat > "$W/bare.org" <<'EOF' +* Steps +Jumps straight in with no orientation section. +EOF + run python3 "$CHECKER" "$W" + [ "$status" -eq 1 ] + [[ "$output" == *"missing-section"* ]] + [[ "$output" == *"bare.org"* ]] +} + +@test "workflow-integrity: a trigger phrase claimed by two workflows is flagged" { + # Make engine also claim alpha's trigger. + sed -i 's/ - Triggers: "run engine"/ - Triggers: "run engine", "do alpha"/' "$W/INDEX.org" + run python3 "$CHECKER" "$W" + [ "$status" -eq 1 ] + [[ "$output" == *"dup-trigger"* ]] + [[ "$output" == *"do alpha"* ]] +} diff --git a/scripts/workflow-integrity.py b/scripts/workflow-integrity.py new file mode 100755 index 0000000..fa33c4c --- /dev/null +++ b/scripts/workflow-integrity.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Integrity checks for the .ai/workflows/ directory and its INDEX.org. + +Startup's drift check catches index-vs-directory mismatches; this goes deeper: +a workflow referencing a renamed script, a plugin whose engine was deleted, a +missing required section, a duplicate trigger phrase. Runs against the canonical +claude-templates/.ai/workflows/ by default; pass a directory to check another. + +Checks: + 1. indexed-or-plugin every *.org (except INDEX.org) is a catalog entry in + INDEX.org, or a source plugin (engine.plugin.org) of an + indexed engine. + 2. indexed-exists every catalog entry points at a file that exists. + 3. script-refs every .ai/scripts/ reference in a workflow resolves + to a real file under the canonical scripts dir. + 4. plugin-parent every engine.plugin.org maps to an indexed engine. + 5. orientation every non-plugin workflow has an orientation section + (Overview / Purpose / When to Use|Run / Status). + 6. trigger-uniqueness no trigger phrase is claimed by two different workflows. + +Exit 0 when clean, 1 when any check fails, 2 on bad usage. + +Usage: + workflow-integrity.py [WORKFLOWS_DIR] +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +DEFAULT_DIR = REPO / "claude-templates" / ".ai" / "workflows" + +CATALOG_RE = re.compile(r"^- =([A-Za-z0-9._-]+\.org)= ", re.M) +PLUGIN_RE = re.compile(r"^(.+)\.[^.]+\.org$") # engine.plugin.org +SCRIPT_REF_RE = re.compile(r"\.ai/scripts/([A-Za-z0-9][A-Za-z0-9._/-]*)") +QUOTED_RE = re.compile(r'"([^"]+)"') +ORIENTATION_RE = re.compile(r"^\* (overview|purpose|when to use|when to run|status)\b", + re.M | re.I) + + +def catalog_entries(index_text: str) -> set[str]: + """Indexed workflow filenames — list items of the form `- =name.org= — ...`. + Distinguishes catalog entries from prose mentions like =todo.org=.""" + return set(CATALOG_RE.findall(index_text)) + + +def scripts_dir_for(workflows_dir: Path) -> Path: + """The scripts dir a workflows dir's .ai/scripts/ references resolve against.""" + return workflows_dir.parent / "scripts" + + +def trigger_map(index_text: str) -> dict[str, set[str]]: + """Map each trigger phrase to the set of workflows that claim it. + + Tracks the current workflow as catalog entries are seen, then attributes + quoted phrases on any subsequent 'trigger' line to it. + """ + phrases: dict[str, set[str]] = {} + current: str | None = None + for line in index_text.splitlines(): + m = CATALOG_RE.match(line) + if m: + current = m.group(1) + continue + if current and "rigger" in line: # "Triggers:", "Full-prep triggers:", etc. + for phrase in QUOTED_RE.findall(line): + phrases.setdefault(phrase, set()).add(current) + return phrases + + +def check(workflows_dir: Path) -> list[str]: + findings: list[str] = [] + index = workflows_dir / "INDEX.org" + if not index.is_file(): + return [f"no INDEX.org in {workflows_dir}"] + index_text = index.read_text(encoding="utf-8") + indexed = catalog_entries(index_text) + engines = {n[:-len(".org")] for n in indexed} + workflows = sorted(p.name for p in workflows_dir.glob("*.org") if p.name != "INDEX.org") + scripts_dir = scripts_dir_for(workflows_dir) + + def is_plugin(name: str) -> bool: + return any(name.startswith(e + ".") and name != e + ".org" for e in engines) + + # 1. indexed-or-plugin + for w in workflows: + if w not in indexed and not is_plugin(w): + findings.append(f"[orphan] {w}: not indexed in INDEX.org and not a plugin of an indexed engine") + + # 2. indexed-exists + for n in sorted(indexed): + if not (workflows_dir / n).is_file(): + findings.append(f"[stale-index] INDEX lists {n} but no such file exists") + + # 4. plugin-parent + for w in workflows: + if w in indexed: + continue + m = PLUGIN_RE.match(w) + if m: + parent = m.group(1) + ".org" + if parent not in indexed: + findings.append(f"[orphan-plugin] {w}: parent engine {parent} is not indexed") + elif not (workflows_dir / parent).is_file(): + findings.append(f"[orphan-plugin] {w}: parent engine {parent} is missing") + + # 3. script-refs + for w in workflows: + text = (workflows_dir / w).read_text(encoding="utf-8") + for ref in sorted(set(SCRIPT_REF_RE.findall(text))): + ref = ref.rstrip(".") + if (scripts_dir / ref).exists(): + continue + if (scripts_dir / ref.split("/")[0]).exists(): + continue + findings.append(f"[bad-ref] {w}: references .ai/scripts/{ref} which does not exist") + + # 5. orientation (plugins are adapters loaded by their engine — exempt) + for w in workflows: + if is_plugin(w): + continue + text = (workflows_dir / w).read_text(encoding="utf-8") + if not ORIENTATION_RE.search(text): + findings.append(f"[missing-section] {w}: no orientation section " + "(Overview / Purpose / When to Use|Run / Status)") + + # 6. trigger-uniqueness + for phrase, owners in sorted(trigger_map(index_text).items()): + if len(owners) > 1: + findings.append(f"[dup-trigger] \"{phrase}\" claimed by {', '.join(sorted(owners))}") + + return findings + + +def main() -> int: + if len(sys.argv) > 2: + print("usage: workflow-integrity.py [WORKFLOWS_DIR]", file=sys.stderr) + return 2 + workflows_dir = Path(sys.argv[1]) if len(sys.argv) == 2 else DEFAULT_DIR + if not workflows_dir.is_dir(): + print(f"workflow-integrity: {workflows_dir} is not a directory", file=sys.stderr) + return 2 + + findings = check(workflows_dir) + n_workflows = len([p for p in workflows_dir.glob("*.org") if p.name != "INDEX.org"]) + if findings: + print("workflow-integrity: FAIL") + for f in findings: + print(f" {f}") + return 1 + print(f"workflow-integrity: OK ({n_workflows} workflows)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- cgit v1.2.3