ersatztv/scripts/tests/test_guard_inventory.py

"""docs/guard-inventory.md covers exactly the guards that exist (ersatztv#774, #775).

THIS FILE IS THE ANSWER TO BOTH ISSUES' "is a mechanical check possible?" QUESTION, and the shape of
the answer matters more than the code.

What is NOT possible, and must not be attempted: a lint that flags filter-shaped guards by matching
`.filter(` / `.Where(` / `grep` inside guard code. #774 asks for this explicitly and the honest
answer is no. The token is not the defect — `ToolCatalogTests.Every_Query_Parameter_Should_Be_A_
Declared_Property` filters correctly eight lines from a completeness assertion that must not. A
matcher would flag both, and a matcher that is wrong half the time is waved through until it is
never read, which is the symptom-keyed-guard mistake this repo has now paid for at #644 and #650.
Worse, it is a string predicate over source, and this repo's own record is that a string predicate
takes three or more rounds to get right (#629, #633, #698). Building it would be #774 violating
#774.

What IS possible, and is what this file does: you cannot mechanically detect that a guard reasons
about a sample, but you CAN mechanically guarantee that every guard has been *classified by someone*
and that its claimed proof exists. That converts both rules from "remember to do this" into "the
suite goes red until you have". Specifically:

  * the guard population is DERIVED — read out of the GIT INDEX for `.claude/hooks/` and
    `.husky/`, plus every `scripts/*.sh|py` referenced by a workflow or a hook — and compared for
    SET EQUALITY against the inventory's rows, in both directions. The index rather than the disk
    per ersatztv#806: a filesystem walk reports build output and editor droppings and differs per
    machine, so it cannot be the authoritative source a completeness claim needs;
  * every row's `Proof ref` is resolved to a real file and a real `def` in it;
  * every row's `Kind` and `Proof` come from a closed vocabulary, so a typo cannot invent a state.

The residue this leaves, named rather than papered over: nothing here checks that a row's
`MUTATION` claim is TRUE. A test named in the table might merely exercise the guard. That judgement
stays with review, and the table is the thing review reads.
"""

from __future__ import annotations

import re
from collections import Counter
from pathlib import Path

from scripts.tests import tracked_files
from scripts.tests.tracked_files import tracked_children, tracked_paths

REPO_ROOT = Path(__file__).resolve().parents[2]
INVENTORY = REPO_ROOT / "docs" / "guard-inventory.md"

# THE POPULATION SCOPES, resolved against the GIT INDEX rather than the filesystem (ersatztv#806).
# Directory + patterns instead of `Path.glob`; `scripts/tests/tracked_files.py` carries why the disk
# is not an authoritative source. `.husky` is the sharp case: it holds an untracked `_/` of 17
# npm-generated shims, and the previous `iterdir() ... if p.is_file()` excluded them only because
# `_` happens to be a directory — by accident, not by design, so the obvious "make it recursive"
# edit would have reintroduced #778's third defect inside the repo's own model guard.
HOOKS = (".claude/hooks", ("*.sh",))
HUSKY = (".husky", ("*",))
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))
GUARD_TESTS = ("scripts/tests", ("test_*.py",))

KINDS = {"GUARD", "TOOLING", "PROOF"}
PROOFS = {"MUTATION", "BEHAVIOUR-ONLY", "NONE"}

# `scripts/x.sh` AND `scripts/tests/x.py`. The first version omitted the `/`, and the consequence was
# not theoretical: the three guard files this inventory shipped with were themselves outside the
# population it checked for completeness, so they acquired no rows and the guard stayed green. Cold
# review found it. A completeness guard blind to its own author's new guards is the defect this
# whole change is about, so the miss is recorded here rather than quietly corrected.
_SCRIPT_REF = re.compile(r"scripts/(?:[a-z0-9_.-]+/)?[a-z0-9_.-]+\.(?:sh|py)")
_ROW = re.compile(r"^\|\s*`([^`]+)`\s*\|([^|]*)\|\s*([A-Z-]+)\s*\|\s*([A-Z-]+)\s*\|([^|]*)\|\s*$", re.M)

# The prose summary, parsed so it cannot drift from the table it summarises. It already had:
# shipped as "28 guards, 4 tooling … 6 … 3 … 19" against a table holding 27/5/6/3/18, because it was
# a hand-maintained mirror with no equality check — #773's Family C inside the deliverable arguing
# against it. Both reviewers caught it independently.
_SUMMARY = re.compile(
    r"(\d+)\s+guards?,\s+(\d+)\s+tooling\s+scripts?,\s+(\d+)\s+proof\s+files?\.\s+"
    r"\*\*(\d+)\s+guards?\s+carry\s+a\s+mutation\s+proof;\s+(\d+)\s+(?:are|is)\s+behaviour-only;\s+"
    r"(\d+)\s+have\s+none\.\*\*"
)


def derived_guard_files() -> set[str]:
    """THE AUTHORITATIVE POPULATION, from the git index and the call sites — never a list.

    Four contributors, unioned: three scope directories plus the paths those files REFERENCE. The
    hook and husky directories are taken whole, so a new hook is in the population the moment it is
    STAGED. The scripts half is discovered by scanning what the workflows and hooks actually
    INVOKE, rather than taking `scripts/` whole — a script nothing calls is not a guard, and taking
    it whole would drag in every helper and make the inventory a chore that gets rubber-stamped.
    Both halves are gated on the index: the callers by `tracked_paths`, the targets they name by the
    `tracked` set below.

    "The moment it is STAGED" rather than "the moment it exists" is the ersatztv#806 change, and
    it is a strengthening: an untracked `foo.sh` dropped in `.claude/hooks/` used to enter this
    population and demand an inventory row for a file that is not part of the repo — red on that
    checkout, green in CI, which is #778's third shape. Nothing weakens, because a guard that is not
    staged is not on its way to anyone else either.
    """
    found = (
        tracked_children(*HOOKS)
        | tracked_children(*HUSKY)
        # `pr-checks.yml` runs `pytest scripts/tests` as a directory, so every file in it is invoked
        # and none is individually named anywhere. Taking the directory whole is the only derivation
        # that matches how they actually run.
        | tracked_children(*GUARD_TESTS)
    )

    # THE REFERENCED TARGETS ARE GATED ON THE INDEX, NOT ON `Path.exists()`. Converting the CALLERS
    # and leaving the members they contribute on a disk check would have left a quarter of this
    # population answering a question about the machine: a tracked workflow naming
    # `scripts/generated/helper.sh` that exists on one laptop only would enter there, demand an
    # inventory row for a file that is not in the repo, and go red on that checkout while CI stayed
    # green — #778's third shape, in the guard this file calls its model.
    #
    # A referenced path that git does not track is therefore dropped silently, and that is the right
    # residual rather than an assertion: `_SCRIPT_REF` matches any occurrence, including inside a
    # comment or an `::error::` string (limit 3 in `docs/guard-inventory.md`), so demanding that
    # every matched path be tracked would redden a correct tree on a prose mention.
    # Called through the MODULE, never `from … import _git_ls_files`. A direct name binding is
    # captured at import time, and the exhaustive proof in
    # `test_guard_populations_derive_from_git.py` then cannot narrow the index for this branch at
    # all — every referenced target reports as surviving removal, which is a red for the wrong
    # reason and, worse, means the branch is untested however the proof reads.
    tracked = set(tracked_files._git_ls_files())
    callers = tracked_paths(*WORKFLOWS) + tracked_paths(*HOOKS) + tracked_paths(*HUSKY)
    for caller in callers:
        for ref in _SCRIPT_REF.findall(caller.read_text()):
            if ref in tracked:
                found.add(ref)
    return found


def wired_hook_files() -> set[str]:
    """Hooks reachable from `.claude/settings.json` or a husky hook — WIRING, not existence.

    Directory membership is not execution. A hook whose settings.json registration is deleted keeps
    its file, keeps its inventory row, and stops running — and the table would go on describing a
    working guard. That is #631 and #719's shape ("wired is not running") one level down, and it was
    the derivation's blind spot until cold review named it.

    COMMENT LINES ARE STRIPPED from the husky hooks first, and that is not a refinement — the first
    version of this function counted a mention anywhere, and `.husky/pre-commit:7` reads

        # CI where a base ref exists). Fail-open shim — see .claude/hooks/decisions-guard.sh.

    one line above the real invocation. Delete line 8 and keep line 7 and the hook would still have
    read as wired, which is the exact substitution of mention for invocation this function exists to
    stop, reproduced inside the fix for it. `.claude/settings.json` needs no stripping: JSON has no
    comments, so every occurrence there is in a real command string.
    """
    text = (REPO_ROOT / ".claude" / "settings.json").read_text()
    for husky in tracked_paths(*HUSKY):
        text += "\n".join(line for line in husky.read_text().splitlines() if not line.lstrip().startswith("#"))
    return {rel for rel in tracked_children(*HOOKS) if rel.rpartition("/")[2] in text}


def inventory_rows() -> list[tuple[str, str, str, str]]:
    """(guard, kind, proof, proof_ref) for each table row."""
    rows = []
    for guard, _blocks, kind, proof, ref in _ROW.findall(INVENTORY.read_text()):
        rows.append((guard.strip(), kind.strip(), proof.strip(), ref.strip()))
    return rows


# ------------------------------------------------------------------------------------------------
# ANTI-VACUITY FIRST — a row regex that stopped matching would make every assertion below compare
# empty sets and report a fully-covered inventory. That is the failure this file exists to prevent,
# so it is checked before anything depends on it.
# ------------------------------------------------------------------------------------------------


def test_the_table_actually_parsed():
    rows = inventory_rows()
    assert len(rows) >= 25, (
        f"only parsed {len(rows)} rows out of {INVENTORY.name} — the row pattern has stopped "
        "matching the table's markdown, so the coverage assertions below are vacuous."
    )
    assert len(derived_guard_files()) >= 25, "the guard discovery walk found almost nothing"


def test_no_duplicate_rows():
    """Two rows for one guard would let one satisfy the set comparison while the other says
    anything at all — including a fabricated proof."""
    guards = [g for g, _, _, _ in inventory_rows()]
    dupes = sorted({g for g in guards if guards.count(g) > 1})
    assert not dupes, f"{INVENTORY.name} has duplicate rows for {dupes}"


# ------------------------------------------------------------------------------------------------
# SET EQUALITY, BOTH DIRECTIONS
# ------------------------------------------------------------------------------------------------


def test_the_inventory_covers_exactly_the_guards_that_exist():
    listed = {g for g, _, _, _ in inventory_rows()}
    found = derived_guard_files()

    missing = sorted(found - listed)
    assert not missing, (
        f"these guard files exist but have no row in {INVENTORY.name}: {missing}. Every guard must "
        "be classified — add a row giving what it blocks, whether it is a GUARD or TOOLING, and "
        "whether it ships a mutation proof. An unclassified guard is one nobody has decided is "
        "load-bearing, which is how #631's suite ran nowhere for months."
    )
    phantom = sorted(listed - found)
    assert not phantom, (
        f"{INVENTORY.name} lists {phantom}, which no longer exist or are no longer invoked by any "
        "workflow or hook. A row for a guard that does not run reads as coverage and is not."
    )


# ------------------------------------------------------------------------------------------------
# THE CLAIMS IN EACH ROW RESOLVE
# ------------------------------------------------------------------------------------------------


def test_every_row_uses_the_closed_vocabulary():
    for guard, kind, proof, _ref in inventory_rows():
        assert kind in KINDS, f"{guard}: Kind {kind!r} is not one of {sorted(KINDS)}"
        assert proof in PROOFS, f"{guard}: Proof {proof!r} is not one of {sorted(PROOFS)}"


def test_every_claimed_proof_names_a_test_that_exists():
    """The half that makes the table load-bearing rather than decorative.

    A row claiming MUTATION with a `Proof ref` that no longer resolves is worse than a row claiming
    NONE: it tells the next reader this guard is covered. Renaming a test then silently converts a
    proven guard into an unproven one that still reads as proven, and nothing else in the repo
    would notice.
    """
    for guard, _kind, proof, ref in inventory_rows():
        if proof == "NONE":
            assert ref in ("—", "-", ""), f"{guard}: Proof is NONE but a ref is given ({ref!r})"
            continue

        assert "::" in ref, f"{guard}: Proof is {proof} but the ref {ref!r} is not file::function"
        filename, func = ref.strip("`").split("::", 1)
        path = REPO_ROOT / "scripts" / "tests" / filename
        assert path.exists(), f"{guard}: proof ref names {filename}, which does not exist"
        assert re.search(rf"^def {re.escape(func)}\(", path.read_text(), re.M), (
            f"{guard}: {filename} has no `def {func}(`. The proof ref is stale — either the test "
            "was renamed (update the row) or it was deleted (this guard is now unproven, and the "
            "row must say NONE)."
        )


def test_every_hook_file_is_actually_WIRED():
    """A hook file nothing registers is dead code holding an inventory row that reads as coverage."""
    staged = tracked_children(*HOOKS)
    unwired = sorted(staged - wired_hook_files())
    assert not unwired, (
        f"these hook files exist and have inventory rows but are referenced by neither "
        f".claude/settings.json nor any .husky/ hook: {unwired}. They do not run. Either wire them "
        "or delete them — a row for a hook that never fires is the coverage claim #631 paid for."
    )


def test_proof_rows_do_not_themselves_claim_a_proof():
    """`PROOF` exists to stop an infinite regress, and the regress is not hypothetical.

    Once `scripts/tests/*.py` entered the population, every mutation proof became a row needing a
    proof of its own, and so on. `PROOF` marks a file whose job IS to prove another guard; grading
    it would ask what proves the prover, forever. Files under `scripts/tests/` that enforce a repo
    invariant with no separate guard file behind them are `GUARD`, not `PROOF`, and are graded
    normally — that is the honest place to draw the line.
    """
    for guard, kind, proof, ref in inventory_rows():
        if kind == "PROOF":
            assert proof == "NONE", f"{guard} is PROOF but claims Proof {proof}"
            assert guard.startswith("scripts/tests/"), f"{guard} is marked PROOF but does not live in scripts/tests/"
            assert ref in ("—", "-", ""), f"{guard}: PROOF rows carry no proof ref"


def test_every_proof_ref_points_at_a_row_marked_PROOF():
    """Ties the two halves of the table together. A guard citing a test that the table does not
    classify as a PROOF means the population and the refs disagree about what that file is."""
    rows = inventory_rows()
    proof_files = {g for g, k, _, _ in rows if k == "PROOF"}
    for guard, _kind, proof, ref in rows:
        if proof == "NONE":
            continue
        filename = ref.strip("`").split("::", 1)[0]
        path = f"scripts/tests/{filename}"
        # A test-file GUARD may cite ITSELF: its mutation cases live in the same file, because the
        # thing it guards is a repo invariant rather than another script. Splitting those into a
        # separate file to satisfy the table would be bookkeeping driving the code.
        if path == guard:
            continue
        assert path in proof_files, (
            f"{guard} cites {filename} as its proof, but that file is not a PROOF row in this "
            "table. The citation and the classification must agree."
        )


def test_the_summary_counts_match_the_table():
    """The prose is DERIVED-checked, not hand-maintained.

    It shipped wrong the first time — "28 guards, 4 tooling … 19 have none" against a table holding
    27/5/…/18 — because it was a mirror with no equality check. Both cold reviewers found it
    independently, which is the clearest possible evidence that a summary nobody checks is a summary
    nobody can trust.
    """
    rows = inventory_rows()
    kinds = Counter(k for _, k, _, _ in rows)
    grades = Counter(p for _, k, p, _ in rows if k == "GUARD")

    m = _SUMMARY.search(INVENTORY.read_text())
    assert m, (
        "could not find the summary sentence in the expected shape. It must read exactly like: "
        "`N guards, N tooling scripts, N proof files. **N guards carry a mutation proof; N are "
        "behaviour-only; N have none.**` — if you reword it, update `_SUMMARY` in the same commit, "
        "because an unparsed summary is an unchecked one."
    )
    claimed = tuple(int(g) for g in m.groups())
    actual = (
        kinds["GUARD"],
        kinds["TOOLING"],
        kinds["PROOF"],
        grades["MUTATION"],
        grades["BEHAVIOUR-ONLY"],
        grades["NONE"],
    )
    assert claimed == actual, (
        f"the summary claims (guards, tooling, proofs, mutation, behaviour-only, none) = {claimed} "
        f"but the table holds {actual}."
    )


def test_tooling_rows_never_claim_a_proof():
    """A TOOLING row asserting nothing cannot have a proof that it can go red, and letting one
    carry a ref would quietly inflate the coverage count at the bottom of the inventory."""
    for guard, kind, proof, _ref in inventory_rows():
        if kind == "TOOLING":
            assert proof == "NONE", f"{guard} is TOOLING but claims Proof {proof}"