ersatztv/scripts/tests/test_guard_inventory.py

"""docs/guard-inventory.md covers exactly the guards that exist (ersatztv#774, #775).

THIS FILE IS THE ANSWER TO BOTH ISSUES' "is a mechanical check possible?" QUESTION, and the shape of
the answer matters more than the code.

What is NOT possible, and must not be attempted: a lint that flags filter-shaped guards by matching
`.filter(` / `.Where(` / `grep` inside guard code. #774 asks for this explicitly and the honest
answer is no. The token is not the defect — `ToolCatalogTests.Every_Query_Parameter_Should_Be_A_
Declared_Property` filters correctly eight lines from a completeness assertion that must not. A
matcher would flag both, and a matcher that is wrong half the time is waved through until it is
never read, which is the symptom-keyed-guard mistake this repo has now paid for at #644 and #650.
Worse, it is a string predicate over source, and this repo's own record is that a string predicate
takes three or more rounds to get right (#629, #633, #698). Building it would be #774 violating
#774.

What IS possible, and is what this file does: you cannot mechanically detect that a guard reasons
about a sample, but you CAN mechanically guarantee that every guard has been *classified by someone*
and that its claimed proof exists. That converts both rules from "remember to do this" into "the
suite goes red until you have". Specifically:

  * the guard population is DERIVED — read out of the GIT INDEX for `.claude/hooks/` and
    `.husky/`, plus every `scripts/*.sh|py` referenced by a workflow or a hook — and compared for
    SET EQUALITY against the inventory's rows, in both directions. The index rather than the disk
    per ersatztv#806: a filesystem walk reports build output and editor droppings and differs per
    machine, so it cannot be the authoritative source a completeness claim needs;
  * every row's `Proof ref` is resolved to a real file and a real `def` in it;
  * every row's `Kind` and `Proof` come from a closed vocabulary, so a typo cannot invent a state.

The residue this leaves, named rather than papered over: nothing here checks that a row's
`MUTATION` claim is TRUE. A test named in the table might merely exercise the guard. That judgement
stays with review, and the table is the thing review reads.
"""

from __future__ import annotations

import re
from collections import Counter
from pathlib import Path

from scripts.tests import tracked_files
from scripts.tests.tracked_files import tracked_children, tracked_paths

REPO_ROOT = Path(__file__).resolve().parents[2]
INVENTORY = REPO_ROOT / "docs" / "guard-inventory.md"

# THE POPULATION SCOPES, resolved against the GIT INDEX rather than the filesystem (ersatztv#806).
# Directory + patterns instead of `Path.glob`; `scripts/tests/tracked_files.py` carries why the disk
# is not an authoritative source. `.husky` is the sharp case: it holds an untracked `_/` of 17
# npm-generated shims, and the previous `iterdir() ... if p.is_file()` excluded them only because
# `_` happens to be a directory — by accident, not by design, so the obvious "make it recursive"
# edit would have reintroduced #778's third defect inside the repo's own model guard.
HOOKS = (".claude/hooks", ("*.sh",))
HUSKY = (".husky", ("*",))
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))
GUARD_TESTS = ("scripts/tests", ("test_*.py",))

KINDS = {"GUARD", "TOOLING", "PROOF"}
PROOFS = {"MUTATION", "BEHAVIOUR-ONLY", "NONE"}

# `scripts/x.sh` AND `scripts/tests/x.py`. Omitting the `/` had a consequence that was not
# theoretical: the three guard files this inventory shipped with were themselves outside the
# population it checked for completeness, so they acquired no rows and the guard stayed green.
# A completeness guard blind to its own author's new guards is the defect this whole change is
# about, so the miss is recorded here rather than quietly corrected.
_SCRIPT_REF = re.compile(r"scripts/(?:[a-z0-9_.-]+/)?[a-z0-9_.-]+\.(?:sh|py)")
_ROW = re.compile(r"^\|\s*`([^`]+)`\s*\|([^|]*)\|\s*([A-Z-]+)\s*\|\s*([A-Z-]+)\s*\|([^|]*)\|\s*$", re.M)

# The prose summary, parsed so it cannot drift from the table it summarises. It already had:
# shipped as "28 guards, 4 tooling … 6 … 3 … 19" against a table holding 27/5/6/3/18, because it was
# a hand-maintained mirror with no equality check — #773's Family C inside the deliverable arguing
# against it.
_SUMMARY = re.compile(
    r"(\d+)\s+guards?,\s+(\d+)\s+tooling\s+scripts?,\s+(\d+)\s+proof\s+files?\.\s+"
    r"\*\*(\d+)\s+guards?\s+carry\s+a\s+mutation\s+proof;\s+(\d+)\s+(?:are|is)\s+behaviour-only;\s+"
    r"(\d+)\s+have\s+none\.\*\*"
)


def derived_guard_files() -> set[str]:
    """THE AUTHORITATIVE POPULATION, from the git index and the call sites — never a list.

    Four contributors, unioned: three scope directories plus the paths those files REFERENCE. The
    hook and husky directories are taken whole, so a new hook is in the population the moment it is
    STAGED. The scripts half is discovered by scanning what the workflows and hooks actually
    INVOKE, rather than taking `scripts/` whole — a script nothing calls is not a guard, and taking
    it whole would drag in every helper and make the inventory a chore that gets rubber-stamped.
    Both halves are gated on the index: the callers by `tracked_paths`, the targets they name by the
    `tracked` set below.

    "The moment it is STAGED" rather than "the moment it exists" is the ersatztv#806 change, and
    it is a strengthening: an untracked `foo.sh` dropped in `.claude/hooks/` used to enter this
    population and demand an inventory row for a file that is not part of the repo — red on that
    checkout, green in CI, which is #778's third shape. Nothing weakens, because a guard that is not
    staged is not on its way to anyone else either.
    """
    found = (
        tracked_children(*HOOKS)
        | tracked_children(*HUSKY)
        # `pr-checks.yml` runs `pytest scripts/tests` as a directory, so every file in it is invoked
        # and none is individually named anywhere. Taking the directory whole is the only derivation
        # that matches how they actually run.
        | tracked_children(*GUARD_TESTS)
    )

    # THE REFERENCED TARGETS ARE GATED ON THE INDEX, NOT ON `Path.exists()`. Converting the CALLERS
    # and leaving the members they contribute on a disk check would have left a quarter of this
    # population answering a question about the machine: a tracked workflow naming
    # `scripts/generated/helper.sh` that exists on one laptop only would enter there, demand an
    # inventory row for a file that is not in the repo, and go red on that checkout while CI stayed
    # green — #778's third shape, in the guard this file calls its model.
    #
    # A referenced path that git does not track is therefore dropped silently, and that is the right
    # residual rather than an assertion: `_SCRIPT_REF` matches any occurrence, including inside a
    # comment or an `::error::` string (limit 3 in `docs/guard-inventory.md`), so demanding that
    # every matched path be tracked would redden a correct tree on a prose mention.
    # Called through the MODULE, never `from … import _git_ls_files`. A direct name binding is
    # captured at import time, and the exhaustive proof in
    # `test_guard_populations_derive_from_git.py` then cannot narrow the index for this branch at
    # all — every referenced target reports as surviving removal, which is a red for the wrong
    # reason and, worse, means the branch is untested however the proof reads.
    tracked = set(tracked_files._git_ls_files())
    callers = tracked_paths(*WORKFLOWS) + tracked_paths(*HOOKS) + tracked_paths(*HUSKY)
    for caller in callers:
        for ref in _SCRIPT_REF.findall(caller.read_text()):
            if ref in tracked:
                found.add(ref)
    return found


def wired_hook_files() -> set[str]:
    """Hooks reachable from `.claude/settings.json` or a husky hook — WIRING, not existence.

    Directory membership is not execution. A hook whose settings.json registration is deleted keeps
    its file, keeps its inventory row, and stops running — and the table would go on describing a
    working guard. That is #631 and #719's shape ("wired is not running") one level down.

    COMMENT LINES ARE STRIPPED from the husky hooks first, and that is not a refinement — counting
    a mention anywhere counts `.husky/pre-commit:7`, which reads

        # CI where a base ref exists). Fail-open shim — see .claude/hooks/decisions-guard.sh.

    one line above the real invocation. Delete line 8 and keep line 7 and the hook would still read
    as wired, which is the exact substitution of mention for invocation this function exists to
    stop. `.claude/settings.json` needs no stripping: JSON has no comments, so every occurrence
    there is in a real command string.
    """
    text = (REPO_ROOT / ".claude" / "settings.json").read_text()
    for husky in tracked_paths(*HUSKY):
        text += "\n".join(line for line in husky.read_text().splitlines() if not line.lstrip().startswith("#"))
    return {rel for rel in tracked_children(*HOOKS) if rel.rpartition("/")[2] in text}


def section(heading: str) -> str:
    """The body of ONE `## ` section of the inventory.

    Scoped since ersatztv#786 added `## Workflow-job guards`, a second five-column table whose
    subject is workflow JOBS rather than guard FILES. `_ROW` is keyed on SHAPE, not on location, so
    an unscoped `findall` over the whole document would pull those rows into the set-equality below
    and report every one of them as a guard file that does not exist. A missing heading raises here
    rather than returning an empty string: a silent empty section is the vacuous-population failure
    this file exists to prevent.
    """
    text = INVENTORY.read_text()
    start = text.index(f"\n## {heading}\n")
    nxt = text.find("\n## ", start + 1)
    return text[start : nxt if nxt != -1 else len(text)]


def inventory_rows() -> list[tuple[str, str, str, str]]:
    """(guard, kind, proof, proof_ref) for each row of the `## Inventory` table."""
    rows = []
    for guard, _blocks, kind, proof, ref in _ROW.findall(section("Inventory")):
        rows.append((guard.strip(), kind.strip(), proof.strip(), ref.strip()))
    return rows


# ------------------------------------------------------------------------------------------------
# ANTI-VACUITY FIRST — a row regex that stopped matching would make every assertion below compare
# empty sets and report a fully-covered inventory. That is the failure this file exists to prevent,
# so it is checked before anything depends on it.
# ------------------------------------------------------------------------------------------------


def test_the_table_actually_parsed():
    rows = inventory_rows()
    assert len(rows) >= 25, (
        f"only parsed {len(rows)} rows out of {INVENTORY.name} — the row pattern has stopped "
        "matching the table's markdown, so the coverage assertions below are vacuous."
    )
    assert len(derived_guard_files()) >= 25, "the guard discovery walk found almost nothing"


def test_no_duplicate_rows():
    """Two rows for one guard would let one satisfy the set comparison while the other says
    anything at all — including a fabricated proof."""
    guards = [g for g, _, _, _ in inventory_rows()]
    dupes = sorted({g for g in guards if guards.count(g) > 1})
    assert not dupes, f"{INVENTORY.name} has duplicate rows for {dupes}"


# ------------------------------------------------------------------------------------------------
# SET EQUALITY, BOTH DIRECTIONS
# ------------------------------------------------------------------------------------------------


def test_the_inventory_covers_exactly_the_guards_that_exist():
    listed = {g for g, _, _, _ in inventory_rows()}
    found = derived_guard_files()

    missing = sorted(found - listed)
    assert not missing, (
        f"these guard files exist but have no row in {INVENTORY.name}: {missing}. Every guard must "
        "be classified — add a row giving what it blocks, whether it is a GUARD or TOOLING, and "
        "whether it ships a mutation proof. An unclassified guard is one nobody has decided is "
        "load-bearing, which is how #631's suite ran nowhere for months."
    )
    phantom = sorted(listed - found)
    assert not phantom, (
        f"{INVENTORY.name} lists {phantom}, which no longer exist or are no longer invoked by any "
        "workflow or hook. A row for a guard that does not run reads as coverage and is not."
    )


# ------------------------------------------------------------------------------------------------
# THE CLAIMS IN EACH ROW RESOLVE
# ------------------------------------------------------------------------------------------------


def test_every_row_uses_the_closed_vocabulary():
    for guard, kind, proof, _ref in inventory_rows():
        assert kind in KINDS, f"{guard}: Kind {kind!r} is not one of {sorted(KINDS)}"
        assert proof in PROOFS, f"{guard}: Proof {proof!r} is not one of {sorted(PROOFS)}"


def test_every_claimed_proof_names_a_test_that_exists():
    """The half that makes the table load-bearing rather than decorative.

    A row claiming MUTATION with a `Proof ref` that no longer resolves is worse than a row claiming
    NONE: it tells the next reader this guard is covered. Renaming a test then silently converts a
    proven guard into an unproven one that still reads as proven, and nothing else in the repo
    would notice.
    """
    for guard, _kind, proof, ref in inventory_rows():
        if proof == "NONE":
            assert ref in ("—", "-", ""), f"{guard}: Proof is NONE but a ref is given ({ref!r})"
            continue

        assert "::" in ref, f"{guard}: Proof is {proof} but the ref {ref!r} is not file::function"
        filename, func = ref.strip("`").split("::", 1)
        path = REPO_ROOT / "scripts" / "tests" / filename
        assert path.exists(), f"{guard}: proof ref names {filename}, which does not exist"
        assert re.search(rf"^def {re.escape(func)}\(", path.read_text(), re.M), (
            f"{guard}: {filename} has no `def {func}(`. The proof ref is stale — either the test "
            "was renamed (update the row) or it was deleted (this guard is now unproven, and the "
            "row must say NONE)."
        )


def test_every_hook_file_is_actually_WIRED():
    """A hook file nothing registers is dead code holding an inventory row that reads as coverage."""
    staged = tracked_children(*HOOKS)
    unwired = sorted(staged - wired_hook_files())
    assert not unwired, (
        f"these hook files exist and have inventory rows but are referenced by neither "
        f".claude/settings.json nor any .husky/ hook: {unwired}. They do not run. Either wire them "
        "or delete them — a row for a hook that never fires is the coverage claim #631 paid for."
    )


def test_proof_rows_do_not_themselves_claim_a_proof():
    """`PROOF` exists to stop an infinite regress, and the regress is not hypothetical.

    Once `scripts/tests/*.py` entered the population, every mutation proof became a row needing a
    proof of its own, and so on. `PROOF` marks a file whose job IS to prove another guard; grading
    it would ask what proves the prover, forever. Files under `scripts/tests/` that enforce a repo
    invariant with no separate guard file behind them are `GUARD`, not `PROOF`, and are graded
    normally — that is the honest place to draw the line.
    """
    for guard, kind, proof, ref in inventory_rows():
        if kind == "PROOF":
            assert proof == "NONE", f"{guard} is PROOF but claims Proof {proof}"
            assert guard.startswith("scripts/tests/"), f"{guard} is marked PROOF but does not live in scripts/tests/"
            assert ref in ("—", "-", ""), f"{guard}: PROOF rows carry no proof ref"


def test_every_proof_ref_points_at_a_row_marked_PROOF():
    """Ties the two halves of the table together. A guard citing a test that the table does not
    classify as a PROOF means the population and the refs disagree about what that file is."""
    rows = inventory_rows()
    proof_files = {g for g, k, _, _ in rows if k == "PROOF"}
    for guard, _kind, proof, ref in rows:
        if proof == "NONE":
            continue
        filename = ref.strip("`").split("::", 1)[0]
        path = f"scripts/tests/{filename}"
        # A test-file GUARD may cite ITSELF: its mutation cases live in the same file, because the
        # thing it guards is a repo invariant rather than another script. Splitting those into a
        # separate file to satisfy the table would be bookkeeping driving the code.
        if path == guard:
            continue
        assert path in proof_files, (
            f"{guard} cites {filename} as its proof, but that file is not a PROOF row in this "
            "table. The citation and the classification must agree."
        )


def test_the_summary_counts_match_the_table():
    """The prose is DERIVED-checked, not hand-maintained.

    A mirror with no equality check shipped "28 guards, 4 tooling … 19 have none" against a table
    holding 27/5/…/18. A summary nobody checks is a summary
    nobody can trust.
    """
    rows = inventory_rows()
    kinds = Counter(k for _, k, _, _ in rows)
    grades = Counter(p for _, k, p, _ in rows if k == "GUARD")

    m = _SUMMARY.search(INVENTORY.read_text())
    assert m, (
        "could not find the summary sentence in the expected shape. It must read exactly like: "
        "`N guards, N tooling scripts, N proof files. **N guards carry a mutation proof; N are "
        "behaviour-only; N have none.**` — if you reword it, update `_SUMMARY` in the same commit, "
        "because an unparsed summary is an unchecked one."
    )
    claimed = tuple(int(g) for g in m.groups())
    actual = (
        kinds["GUARD"],
        kinds["TOOLING"],
        kinds["PROOF"],
        grades["MUTATION"],
        grades["BEHAVIOUR-ONLY"],
        grades["NONE"],
    )
    assert claimed == actual, (
        f"the summary claims (guards, tooling, proofs, mutation, behaviour-only, none) = {claimed} "
        f"but the table holds {actual}."
    )


def test_tooling_rows_never_claim_a_proof():
    """A TOOLING row asserting nothing cannot have a proof that it can go red, and letting one
    carry a ref would quietly inflate the coverage count at the bottom of the inventory."""
    for guard, kind, proof, _ref in inventory_rows():
        if kind == "TOOLING":
            assert proof == "NONE", f"{guard} is TOOLING but claims Proof {proof}"