Files
ersatztv/scripts/tests/test_build_catalog_check_path.py
T
timothyandClaude Opus 5 f2240b0205 test(785): drive build_decisions_catalog's --check path, and its __main__ wiring
`--check` is what the decisions-guard job runs; the existing tests call `render_catalog()`
directly and never call `main()` at all. So neither the stale-detection comparison nor the
`raise SystemExit(main())` wiring had any coverage.

Two mutations, both witnessed red against the real script:

  * disarming `want.strip() != have.strip()` alone -> 3 failed, 3 passed
  * replacing `raise SystemExit(main())` with a bare `main()` -> 1 failed, 5 passed, and
    only the subprocess case reddens. That mutant prints
    `build_decisions_catalog: docs/decisions/README.md is stale` on stderr and exits 0.
    The workflow step reads nothing but the exit code, so CI would report green over a
    stale catalog — the #751/#719 shape the decision record names.

The subprocess case runs the command DERIVED from pr-checks.yml rather than a restated
copy of it, and runs it twice: once on an untouched copy of the corpus, which must exit 0.
Without that half, any failure of the harness — an import error, a missing file in the
copy — would also exit non-zero and read as the guard firing.

Refs: #785
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-21 21:29:00 +02:00

248 lines
12 KiB
Python

"""`scripts/build_decisions_catalog.py`'s `--check` path — the one CI actually runs.
`decisions-guard` (`.gitea/workflows/pr-checks.yml`, the "Active catalog in sync" step) runs
`python3 scripts/build_decisions_catalog.py --check`. `scripts/tests/test_build_catalog.py` covers
`render_catalog()` directly and never calls `main()` at all, so nothing there proves:
* that `main()`'s stale-detection comparison (`want.strip() != have.strip()`) is load-bearing —
a version that always agreed would pass every existing test;
* that the `if __name__ == "__main__": raise SystemExit(main())` wiring actually turns a stale
catalog into a non-zero process exit code, which is the only thing CI's `run:` step reads.
`testing.guard-ships-with-mutation-proof` (#775) is explicit that a guard is not tested because a
test *involving* it passes: it ships with a mutation proof — disarm the guard's own clause, alone,
and a named test must go red. This file is that proof for the catalog-guard, plus the subprocess
proof that the `__main__` wiring is connected (the #751/#719 shape the decision record names: a
green `main()` behind dead wiring).
"""
from __future__ import annotations
import importlib.util
import subprocess
import sys
from pathlib import Path
import scripts.build_decisions_catalog as bc
import scripts.decisions_lib as dl
REPO_ROOT = Path(__file__).resolve().parents[2]
CATALOG_SCRIPT = REPO_ROOT / "scripts" / "build_decisions_catalog.py"
DECISIONS_LIB = REPO_ROOT / "scripts" / "decisions_lib.py"
SCRIPTS_INIT = REPO_ROOT / "scripts" / "__init__.py"
def _current_catalog_text() -> str:
return bc.render_catalog(dl.all_active_records())
# ------------------------------------------------------------------------------------------------
# ANTI-VACUITY — if the real corpus is empty, every assertion below passes for nothing.
# ------------------------------------------------------------------------------------------------
def test_the_real_corpus_is_non_empty_and_renders_a_real_catalog():
records = dl.all_active_records()
active = [r for r in records if r.status == "active" and r.key]
assert active, "no active decision records were parsed — every test below would be vacuous"
text = _current_catalog_text()
assert bc.BANNER in text, "render_catalog produced no banner — not a real catalog document"
assert f"`{active[0].key}`" in text, (
"render_catalog produced no row for a known active record — not a real catalog document"
)
# ------------------------------------------------------------------------------------------------
# `main(["--check"])` — the comparison CI reads
# ------------------------------------------------------------------------------------------------
def test_check_returns_0_when_OUTPUT_matches_render_catalog(tmp_path, monkeypatch):
fresh = _current_catalog_text()
output = tmp_path / "README.md"
output.write_text(fresh.rstrip("\n") + "\n", encoding="utf-8")
monkeypatch.setattr(bc, "OUTPUT", output)
assert bc.main(["--check"]) == 0
def test_check_returns_1_when_OUTPUT_is_stale(tmp_path, monkeypatch):
fresh = _current_catalog_text()
output = tmp_path / "README.md"
# Append a line: the on-disk file no longer matches what render_catalog would produce.
output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
monkeypatch.setattr(bc, "OUTPUT", output)
assert bc.main(["--check"]) == 1
def test_generate_then_check_round_trips(tmp_path, monkeypatch):
"""The no-argument path WRITES the catalog, and a subsequent --check must then pass.
This pins the property CI depends on: generate and check agree. If they ever diverged, `main([])`
would produce a file that `main(["--check"])` immediately rejects — a self-contradiction that
would make the generator useless for fixing the exact problem `--check` reports.
"""
output = tmp_path / "README.md"
assert not output.exists()
monkeypatch.setattr(bc, "OUTPUT", output)
assert bc.main([]) == 0
assert output.exists(), "main([]) with no --check must write OUTPUT"
written = output.read_text(encoding="utf-8")
assert written.strip() == _current_catalog_text().strip()
assert bc.main(["--check"]) == 0, "the file main([]) just wrote must satisfy main(['--check'])"
# ------------------------------------------------------------------------------------------------
# THE `__main__` WIRING — proof CI's subprocess invocation actually surfaces staleness
# ------------------------------------------------------------------------------------------------
def _seed_decisions_copy(root: Path) -> None:
"""Copy only what build_decisions_catalog.py + decisions_lib.py need to resolve a real corpus."""
(root / "scripts").mkdir(parents=True)
(root / "scripts" / "__init__.py").write_bytes(SCRIPTS_INIT.read_bytes())
(root / "scripts" / "build_decisions_catalog.py").write_bytes(CATALOG_SCRIPT.read_bytes())
(root / "scripts" / "decisions_lib.py").write_bytes(DECISIONS_LIB.read_bytes())
docs = root / "docs"
docs.mkdir()
(docs / "decisions.md").write_bytes((REPO_ROOT / "docs" / "decisions.md").read_bytes())
dst_decisions = docs / "decisions"
src_decisions = REPO_ROOT / "docs" / "decisions"
dst_decisions.mkdir()
for item in src_decisions.iterdir():
if item.is_dir():
_copy_tree(item, dst_decisions / item.name)
else:
(dst_decisions / item.name).write_bytes(item.read_bytes())
def _copy_tree(src: Path, dst: Path) -> None:
dst.mkdir(parents=True, exist_ok=True)
for item in src.rglob("*"):
rel = item.relative_to(src)
target = dst / rel
if item.is_dir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(item.read_bytes())
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "pr-checks.yml"
def _ci_check_command() -> list[str]:
"""The command the `decisions-guard` job runs, DERIVED from the workflow, not restated here.
`testing.guard-derives-population-from-source`: a hand-copied command is a second copy of the
workflow that drifts silently, and this test's entire value is that it runs what CI runs. If the
job is reworded, this fails loudly instead of proving the wiring of a command nobody invokes.
"""
for line in WORKFLOW.read_text().splitlines():
if "build_decisions_catalog.py" in line and "--check" in line:
body = line.split("run:", 1)[1].strip() if "run:" in line else line.strip()
assert body.startswith("PYTHONPATH=."), (
f"the decisions-guard invocation no longer starts with PYTHONPATH=.: {body!r}"
)
return ["bash", "-c", body.replace("python3", sys.executable, 1)]
raise AssertionError(
"no `build_decisions_catalog.py --check` line found in pr-checks.yml — either the guard "
"stopped running in CI (which is the more serious finding) or this extractor has drifted"
)
def test_CLI_subprocess_exits_nonzero_on_a_stale_catalog(tmp_path):
"""The real CI invocation, as a subprocess, against a real corpus.
This is the `__main__` → `SystemExit(main())` wiring proof: `main()` returning 1 is worthless if
the process still exits 0, and the workflow step reads nothing but the exit code. It is the
#751/#719 shape the decision record names — a green result behind wiring that is not connected.
The FRESH half is not optional decoration. Without it this test passes whenever the subprocess
dies for any reason at all — an import error, a missing file in the copy, a syntax error — none
of which is the guard detecting anything. `arbitrary-sample-gives-false-negatives` in reverse:
a non-zero exit is only evidence when the same harness is shown to exit zero on a clean corpus.
"""
repo = tmp_path / "repo-copy"
repo.mkdir()
_seed_decisions_copy(repo)
cmd = _ci_check_command()
fresh = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
assert fresh.returncode == 0, (
"the copied corpus does not even pass --check when untouched, so a non-zero exit below "
f"would be the harness failing rather than the guard firing: {fresh.stdout!r} {fresh.stderr!r}"
)
readme = repo / "docs" / "decisions" / "README.md"
readme.write_text(readme.read_text(encoding="utf-8") + "\nSTALE INJECTED LINE\n", encoding="utf-8")
stale = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
assert stale.returncode != 0, (
"the CLI wiring did not surface a stale catalog as a non-zero exit — CI would report "
f"green over a stale README.md: stdout={stale.stdout!r} stderr={stale.stderr!r}"
)
# ------------------------------------------------------------------------------------------------
# THE MUTATION PROOF — disarm `main()`'s stale-detection clause alone, --check must stop detecting
# ------------------------------------------------------------------------------------------------
def _load_mutated_module(tmp_path: Path, mutated_source: str):
mutated_path = tmp_path / "mutated_build_decisions_catalog.py"
mutated_path.write_text(mutated_source, encoding="utf-8")
spec = importlib.util.spec_from_file_location("mutated_build_decisions_catalog", mutated_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_MUTATION_disarming_the_stale_comparison_stops_detection(tmp_path, monkeypatch):
"""`testing.guard-ships-with-mutation-proof` (#775).
`main()`'s ONLY stale-detection logic is the clause `want.strip() != have.strip()`. Disarm that
clause alone — replace it with the constant `False` in an isolated copy of the module, changing
nothing else — and `--check` must stop reporting staleness. If it still returns 1, the deny is
coming from somewhere other than the clause the guard is supposed to hang on, and every test
above proves nothing about it.
"""
text = CATALOG_SCRIPT.read_text(encoding="utf-8")
clause = "want.strip() != have.strip()"
assert clause in text, (
"the stale-detection clause has moved or been reworded; RETARGET this mutation at its new "
"location rather than loosening the string match — a mutation that silently stops mutating "
"is the exact failure this file exists to catch"
)
mutated_source = text.replace(clause, "False", 1)
assert mutated_source != text and clause not in mutated_source, (
"the replacement did not change the source, so the mutant is the subject"
)
fresh = _current_catalog_text()
output = tmp_path / "README.md"
output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
# POSITIVE CONTROL FIRST. Without this, the mutation assertion below would pass just as well if
# `main(["--check"])` never detected anything at all on this input — "the mutant is silent"
# proves nothing unless the real guard is first shown to be loud on the exact same input.
monkeypatch.setattr(bc, "OUTPUT", output)
assert bc.main(["--check"]) == 1, (
"the UNMUTATED guard did not detect the staleness on this input, so a silent mutant below "
"would prove nothing about the clause"
)
mutant = _load_mutated_module(tmp_path, mutated_source)
mutant.OUTPUT = output
assert mutant.main(["--check"]) == 0, (
"disarming `want.strip() != have.strip()` alone did not stop --check from reporting "
"staleness, so that clause is not what the guard's exit code hangs on"
)