`--check` is what the decisions-guard job runs; the existing tests call `render_catalog()`
directly and never call `main()` at all. So neither the stale-detection comparison nor the
`raise SystemExit(main())` wiring had any coverage.
Two mutations, both witnessed red against the real script:
* disarming `want.strip() != have.strip()` alone -> 3 failed, 3 passed
* replacing `raise SystemExit(main())` with a bare `main()` -> 1 failed, 5 passed, and
only the subprocess case reddens. That mutant prints
`build_decisions_catalog: docs/decisions/README.md is stale` on stderr and exits 0.
The workflow step reads nothing but the exit code, so CI would report green over a
stale catalog — the #751/#719 shape the decision record names.
The subprocess case runs the command DERIVED from pr-checks.yml rather than a restated
copy of it, and runs it twice: once on an untouched copy of the corpus, which must exit 0.
Without that half, any failure of the harness — an import error, a missing file in the
copy — would also exit non-zero and read as the guard firing.
Refs: #785
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
248 lines
12 KiB
Python
248 lines
12 KiB
Python
"""`scripts/build_decisions_catalog.py`'s `--check` path — the one CI actually runs.
|
|
|
|
`decisions-guard` (`.gitea/workflows/pr-checks.yml`, the "Active catalog in sync" step) runs
|
|
`python3 scripts/build_decisions_catalog.py --check`. `scripts/tests/test_build_catalog.py` covers
|
|
`render_catalog()` directly and never calls `main()` at all, so nothing there proves:
|
|
|
|
* that `main()`'s stale-detection comparison (`want.strip() != have.strip()`) is load-bearing —
|
|
a version that always agreed would pass every existing test;
|
|
* that the `if __name__ == "__main__": raise SystemExit(main())` wiring actually turns a stale
|
|
catalog into a non-zero process exit code, which is the only thing CI's `run:` step reads.
|
|
|
|
`testing.guard-ships-with-mutation-proof` (#775) is explicit that a guard is not tested because a
|
|
test *involving* it passes: it ships with a mutation proof — disarm the guard's own clause, alone,
|
|
and a named test must go red. This file is that proof for the catalog-guard, plus the subprocess
|
|
proof that the `__main__` wiring is connected (the #751/#719 shape the decision record names: a
|
|
green `main()` behind dead wiring).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import scripts.build_decisions_catalog as bc
|
|
import scripts.decisions_lib as dl
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
CATALOG_SCRIPT = REPO_ROOT / "scripts" / "build_decisions_catalog.py"
|
|
DECISIONS_LIB = REPO_ROOT / "scripts" / "decisions_lib.py"
|
|
SCRIPTS_INIT = REPO_ROOT / "scripts" / "__init__.py"
|
|
|
|
|
|
def _current_catalog_text() -> str:
|
|
return bc.render_catalog(dl.all_active_records())
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------------
|
|
# ANTI-VACUITY — if the real corpus is empty, every assertion below passes for nothing.
|
|
# ------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_real_corpus_is_non_empty_and_renders_a_real_catalog():
|
|
records = dl.all_active_records()
|
|
active = [r for r in records if r.status == "active" and r.key]
|
|
assert active, "no active decision records were parsed — every test below would be vacuous"
|
|
|
|
text = _current_catalog_text()
|
|
assert bc.BANNER in text, "render_catalog produced no banner — not a real catalog document"
|
|
assert f"`{active[0].key}`" in text, (
|
|
"render_catalog produced no row for a known active record — not a real catalog document"
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------------
|
|
# `main(["--check"])` — the comparison CI reads
|
|
# ------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_check_returns_0_when_OUTPUT_matches_render_catalog(tmp_path, monkeypatch):
|
|
fresh = _current_catalog_text()
|
|
output = tmp_path / "README.md"
|
|
output.write_text(fresh.rstrip("\n") + "\n", encoding="utf-8")
|
|
monkeypatch.setattr(bc, "OUTPUT", output)
|
|
|
|
assert bc.main(["--check"]) == 0
|
|
|
|
|
|
def test_check_returns_1_when_OUTPUT_is_stale(tmp_path, monkeypatch):
|
|
fresh = _current_catalog_text()
|
|
output = tmp_path / "README.md"
|
|
# Append a line: the on-disk file no longer matches what render_catalog would produce.
|
|
output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
|
|
monkeypatch.setattr(bc, "OUTPUT", output)
|
|
|
|
assert bc.main(["--check"]) == 1
|
|
|
|
|
|
def test_generate_then_check_round_trips(tmp_path, monkeypatch):
|
|
"""The no-argument path WRITES the catalog, and a subsequent --check must then pass.
|
|
|
|
This pins the property CI depends on: generate and check agree. If they ever diverged, `main([])`
|
|
would produce a file that `main(["--check"])` immediately rejects — a self-contradiction that
|
|
would make the generator useless for fixing the exact problem `--check` reports.
|
|
"""
|
|
output = tmp_path / "README.md"
|
|
assert not output.exists()
|
|
monkeypatch.setattr(bc, "OUTPUT", output)
|
|
|
|
assert bc.main([]) == 0
|
|
assert output.exists(), "main([]) with no --check must write OUTPUT"
|
|
written = output.read_text(encoding="utf-8")
|
|
assert written.strip() == _current_catalog_text().strip()
|
|
|
|
assert bc.main(["--check"]) == 0, "the file main([]) just wrote must satisfy main(['--check'])"
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------------
|
|
# THE `__main__` WIRING — proof CI's subprocess invocation actually surfaces staleness
|
|
# ------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def _seed_decisions_copy(root: Path) -> None:
|
|
"""Copy only what build_decisions_catalog.py + decisions_lib.py need to resolve a real corpus."""
|
|
(root / "scripts").mkdir(parents=True)
|
|
(root / "scripts" / "__init__.py").write_bytes(SCRIPTS_INIT.read_bytes())
|
|
(root / "scripts" / "build_decisions_catalog.py").write_bytes(CATALOG_SCRIPT.read_bytes())
|
|
(root / "scripts" / "decisions_lib.py").write_bytes(DECISIONS_LIB.read_bytes())
|
|
|
|
docs = root / "docs"
|
|
docs.mkdir()
|
|
(docs / "decisions.md").write_bytes((REPO_ROOT / "docs" / "decisions.md").read_bytes())
|
|
|
|
dst_decisions = docs / "decisions"
|
|
src_decisions = REPO_ROOT / "docs" / "decisions"
|
|
dst_decisions.mkdir()
|
|
for item in src_decisions.iterdir():
|
|
if item.is_dir():
|
|
_copy_tree(item, dst_decisions / item.name)
|
|
else:
|
|
(dst_decisions / item.name).write_bytes(item.read_bytes())
|
|
|
|
|
|
def _copy_tree(src: Path, dst: Path) -> None:
|
|
dst.mkdir(parents=True, exist_ok=True)
|
|
for item in src.rglob("*"):
|
|
rel = item.relative_to(src)
|
|
target = dst / rel
|
|
if item.is_dir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(item.read_bytes())
|
|
|
|
|
|
WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "pr-checks.yml"
|
|
|
|
|
|
def _ci_check_command() -> list[str]:
|
|
"""The command the `decisions-guard` job runs, DERIVED from the workflow, not restated here.
|
|
|
|
`testing.guard-derives-population-from-source`: a hand-copied command is a second copy of the
|
|
workflow that drifts silently, and this test's entire value is that it runs what CI runs. If the
|
|
job is reworded, this fails loudly instead of proving the wiring of a command nobody invokes.
|
|
"""
|
|
for line in WORKFLOW.read_text().splitlines():
|
|
if "build_decisions_catalog.py" in line and "--check" in line:
|
|
body = line.split("run:", 1)[1].strip() if "run:" in line else line.strip()
|
|
assert body.startswith("PYTHONPATH=."), (
|
|
f"the decisions-guard invocation no longer starts with PYTHONPATH=.: {body!r}"
|
|
)
|
|
return ["bash", "-c", body.replace("python3", sys.executable, 1)]
|
|
raise AssertionError(
|
|
"no `build_decisions_catalog.py --check` line found in pr-checks.yml — either the guard "
|
|
"stopped running in CI (which is the more serious finding) or this extractor has drifted"
|
|
)
|
|
|
|
|
|
def test_CLI_subprocess_exits_nonzero_on_a_stale_catalog(tmp_path):
|
|
"""The real CI invocation, as a subprocess, against a real corpus.
|
|
|
|
This is the `__main__` → `SystemExit(main())` wiring proof: `main()` returning 1 is worthless if
|
|
the process still exits 0, and the workflow step reads nothing but the exit code. It is the
|
|
#751/#719 shape the decision record names — a green result behind wiring that is not connected.
|
|
|
|
The FRESH half is not optional decoration. Without it this test passes whenever the subprocess
|
|
dies for any reason at all — an import error, a missing file in the copy, a syntax error — none
|
|
of which is the guard detecting anything. `arbitrary-sample-gives-false-negatives` in reverse:
|
|
a non-zero exit is only evidence when the same harness is shown to exit zero on a clean corpus.
|
|
"""
|
|
repo = tmp_path / "repo-copy"
|
|
repo.mkdir()
|
|
_seed_decisions_copy(repo)
|
|
cmd = _ci_check_command()
|
|
|
|
fresh = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
|
|
assert fresh.returncode == 0, (
|
|
"the copied corpus does not even pass --check when untouched, so a non-zero exit below "
|
|
f"would be the harness failing rather than the guard firing: {fresh.stdout!r} {fresh.stderr!r}"
|
|
)
|
|
|
|
readme = repo / "docs" / "decisions" / "README.md"
|
|
readme.write_text(readme.read_text(encoding="utf-8") + "\nSTALE INJECTED LINE\n", encoding="utf-8")
|
|
|
|
stale = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
|
|
assert stale.returncode != 0, (
|
|
"the CLI wiring did not surface a stale catalog as a non-zero exit — CI would report "
|
|
f"green over a stale README.md: stdout={stale.stdout!r} stderr={stale.stderr!r}"
|
|
)
|
|
|
|
|
|
# ------------------------------------------------------------------------------------------------
|
|
# THE MUTATION PROOF — disarm `main()`'s stale-detection clause alone, --check must stop detecting
|
|
# ------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def _load_mutated_module(tmp_path: Path, mutated_source: str):
|
|
mutated_path = tmp_path / "mutated_build_decisions_catalog.py"
|
|
mutated_path.write_text(mutated_source, encoding="utf-8")
|
|
spec = importlib.util.spec_from_file_location("mutated_build_decisions_catalog", mutated_path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_MUTATION_disarming_the_stale_comparison_stops_detection(tmp_path, monkeypatch):
|
|
"""`testing.guard-ships-with-mutation-proof` (#775).
|
|
|
|
`main()`'s ONLY stale-detection logic is the clause `want.strip() != have.strip()`. Disarm that
|
|
clause alone — replace it with the constant `False` in an isolated copy of the module, changing
|
|
nothing else — and `--check` must stop reporting staleness. If it still returns 1, the deny is
|
|
coming from somewhere other than the clause the guard is supposed to hang on, and every test
|
|
above proves nothing about it.
|
|
"""
|
|
text = CATALOG_SCRIPT.read_text(encoding="utf-8")
|
|
clause = "want.strip() != have.strip()"
|
|
assert clause in text, (
|
|
"the stale-detection clause has moved or been reworded; RETARGET this mutation at its new "
|
|
"location rather than loosening the string match — a mutation that silently stops mutating "
|
|
"is the exact failure this file exists to catch"
|
|
)
|
|
mutated_source = text.replace(clause, "False", 1)
|
|
assert mutated_source != text and clause not in mutated_source, (
|
|
"the replacement did not change the source, so the mutant is the subject"
|
|
)
|
|
|
|
fresh = _current_catalog_text()
|
|
output = tmp_path / "README.md"
|
|
output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
|
|
|
|
# POSITIVE CONTROL FIRST. Without this, the mutation assertion below would pass just as well if
|
|
# `main(["--check"])` never detected anything at all on this input — "the mutant is silent"
|
|
# proves nothing unless the real guard is first shown to be loud on the exact same input.
|
|
monkeypatch.setattr(bc, "OUTPUT", output)
|
|
assert bc.main(["--check"]) == 1, (
|
|
"the UNMUTATED guard did not detect the staleness on this input, so a silent mutant below "
|
|
"would prove nothing about the clause"
|
|
)
|
|
|
|
mutant = _load_mutated_module(tmp_path, mutated_source)
|
|
mutant.OUTPUT = output
|
|
assert mutant.main(["--check"]) == 0, (
|
|
"disarming `want.strip() != have.strip()` alone did not stop --check from reporting "
|
|
"staleness, so that clause is not what the guard's exit code hangs on"
|
|
)
|