ersatztv/scripts/tests/test_mutation_harness.py

"""Every `MUTATION` row of `docs/guard-inventory.md` is EXECUTED, not asserted (ersatztv#790).

The `MUTATION` grade means "a clause-level mutation was executed and this named test was witnessed
red". Witnessed once, by hand, that is evidence about the day the row was written and nothing else:
it decays as soon as the guard is edited, and a wrong grade has no way to announce itself.

So this file re-runs every one of them: for each declared mutation in `mutation_manifest.py`, apply
it to an isolated copy of this repository and require the row's OWN named test to go red.

WHAT A GREEN RUN HERE DOES AND DOES NOT PROVE — stated because a mutation harness that overclaims is
the same defect one level up:

  * It proves the recorded proof ref names a test that EXISTS, still collects, and still reacts to
    the declared clause. That is precisely the decay #790 was filed about.
  * It proves the declared clause still occurs, exactly once, in the entry's declared `target` —
    which is not always the guard's own file. A clause that has been reworded fails here rather than
    silently mutating nothing.
  * Combined with the positive control below, it proves every named proof test is GREEN on the
    unmutated sandbox — so "the mutation was noticed" cannot be confused with "the test was already
    red". For the proof tests that perform a disarm of their own, that control also runs the disarm;
    the two named tests that are plain production set-equality checks have no disarm of their own,
    and this harness supplies theirs.
  * It does NOT prove the declared clause is the ONLY thing the guard hangs on. Some entries redden
    through the proof test's own "the clause has moved, RETARGET this" assertion rather than through
    changed behaviour. That is the intended reading, not a hole: those tests perform their
    behavioural disarm themselves on every green run, and what they could not do is notice their own
    clause reference going stale. Which entries those are is a dated measurement and lives in
    `docs/decisions/records/testing/mutation-claims-are-executed.md`, not here.

The `granularity` column is where this file refuses to flatter itself. See `Mutation` in
`mutation_harness_lib.py`: all but one guard admits a single-clause mutation, and the one that does
not CARRIES the finer mutation that survived, which is re-run and required to keep surviving.
"""

from __future__ import annotations

import importlib.util
import re
import shutil
import sys
from dataclasses import replace
from pathlib import Path

import pytest

from scripts.tests.mutation_harness_lib import (
    _BASELINES,
    Mutation,
    _git,
    build_sandbox,
    reset_sandbox,
    run_pytest,
    verify_mutation,
)
from scripts.tests.mutation_manifest import MUTATIONS, UNDECLARED

REPO_ROOT = Path(__file__).resolve().parents[2]
INVENTORY = REPO_ROOT / "docs" / "guard-inventory.md"

# `| `guard` | blocks | kind | proof | proof ref |` — the five-column inventory row. The column
# count is what excludes the two-column tables elsewhere in the file; `test_the_inventory_rows_parse`
# below is the floor that catches the shape changing under it.
_ROW = re.compile(r"^\|\s*`([^`]+)`\s*\|([^|]*)\|\s*(\w[\w-]*)\s*\|\s*([A-Z-]+)\s*\|\s*(.*?)\s*\|\s*$", re.M)


def _inventory_rows() -> list[tuple[str, str, str, str]]:
    """(guard, kind, proof grade, proof ref) for every inventory row.

    Derived from the document the grades live in, never hand-listed: the whole point is that a row
    cannot change its grade without this file noticing.
    """
    rows = [(m.group(1), m.group(3), m.group(4), m.group(5).strip("`")) for m in _ROW.finditer(INVENTORY.read_text())]
    guards = [g for g, _k, _p, _r in rows]
    duplicates = sorted({g for g in guards if guards.count(g) > 1})
    assert not duplicates, (
        f"the inventory lists {duplicates} more than once. Every comparison below reduces rows "
        "through a set or a dict, so a duplicate row is INVISIBLE to this file — it would report all "
        "entries verified over a table that contradicts itself. `test_guard_inventory.py` catches "
        "this too, but only when the whole suite runs, and this file is routinely run alone."
    )
    assert len(rows) >= 50, (
        f"only parsed {len(rows)} inventory rows — the table's shape has changed and this regex now "
        "reads a fraction of it. Every set comparison below would then pass over a population that "
        "is mostly missing, which is the vacuous-completeness failure the inventory itself exists to "
        "prevent."
    )
    return rows


# ------------------------------------------------------------------------------------------------
# THE MANIFEST IS PINNED TO THE INVENTORY — a grade cannot change without this file changing
# ------------------------------------------------------------------------------------------------


def test_the_manifest_covers_exactly_the_MUTATION_rows():
    """Set equality, both directions, because they are different defects.

    A `MUTATION` row with no manifest entry is a claim nothing checks — the state #790 was filed
    about. A manifest entry for a row that is no longer graded `MUTATION` is a check whose subject
    has moved out from under it, and it would keep passing.
    """
    graded = {guard for guard, _kind, grade, _ref in _inventory_rows() if grade == "MUTATION"}
    declared = {m.guard for m in MUTATIONS}
    assert graded, "no row in the inventory is graded MUTATION, so this whole file would prove nothing"
    assert declared - graded == set(), (
        f"declared mutations for rows that are not graded MUTATION: {sorted(declared - graded)}"
    )
    assert graded - declared == set(), (
        f"these rows claim a MUTATION proof with nothing executing it: {sorted(graded - declared)}. "
        "Declare the clause in mutation_manifest.py, or regrade the row."
    )
    assert len(MUTATIONS) == len(declared), "two manifest entries name the same guard"


def test_every_declared_mutation_names_the_row_s_OWN_proof_ref():
    """The manifest may not point at a different test than the row does.

    Without this, the inventory could keep citing a stale proof while the harness quietly exercised
    a healthier one, and the row would read as verified.
    """
    refs = {guard: ref for guard, _kind, grade, ref in _inventory_rows() if grade == "MUTATION"}
    wrong = [(m.guard, m.proof, refs.get(m.guard)) for m in MUTATIONS if refs.get(m.guard) != m.proof]
    assert not wrong, f"manifest proof ref disagrees with the inventory row: {wrong}"


def test_every_GUARD_row_is_either_DECLARED_or_STATED_here():
    """`Done-when`: guards whose mutation cannot be declared are STATED, not silently skipped.

    Set equality against the inventory's GUARD rows, both directions, keyed on the guard. Keying on
    the row's GRADE instead would be cheaper and tautological — a new guard graded NONE would inherit
    a reason automatically and nobody would look at it — and a COUNT moves only on net change, so one
    guard arriving as another is promoted leaves it unchanged. This is the same hand-maintained,
    machine-checked shape as `docs/guard-inventory.md` itself, which is what makes it safe.

    The partition covers `Kind == GUARD` rows only. `TOOLING` asserts nothing and `PROOF` files exist
    to prove other guards, so neither carries a mutation claim to verify — a rule about the Kind
    column rather than a list anyone maintains.
    """
    guards = {guard for guard, kind, _grade, _ref in _inventory_rows() if kind == "GUARD"}
    declared = {m.guard for m in MUTATIONS}
    stated = set(UNDECLARED)

    assert not (declared & stated), (
        f"these guards are both declared and stated as undeclared: {sorted(declared & stated)}"
    )
    unaccounted = sorted(guards - declared - stated)
    assert not unaccounted, (
        f"these guards are neither declared nor stated: {unaccounted}. Declare the clause in "
        "mutation_manifest.py, or write a line in UNDECLARED saying what a proof would need — "
        "silence is the one option that is not available."
    )
    orphaned = sorted(stated - guards)
    assert not orphaned, (
        f"UNDECLARED names rows that are not GUARD-kind rows any more: {orphaned}. A reason for a "
        "guard that no longer exists reads as coverage."
    )
    thin = [g for g, reason in UNDECLARED.items() if len(reason.strip()) < 60]
    assert not thin, f"these UNDECLARED entries say nothing a reader could act on: {thin}"


def test_every_entry_declares_a_known_granularity_and_DETECTOR_entries_CARRY_their_survivor():
    """`DETECTOR` is an admission, and an unevidenced one would be a grading curve.

    #790's complaint about `pin_population_faults` — that neutering a whole helper is "coarse enough
    that a single surviving clause would not be noticed" — applies to every entry graded here. So an
    entry may only claim `DETECTOR` while carrying the finer mutation that was tried, as data rather
    than as a sentence: `test_every_SURVIVING_clause_mutation_still_does` then runs it.
    """
    bad = [m.guard for m in MUTATIONS if m.granularity not in (Mutation.CLAUSE, Mutation.DETECTOR)]
    assert not bad, f"unknown granularity on {bad}; allowed: CLAUSE, DETECTOR"
    assert all(m.why.strip() for m in MUTATIONS), "every declared mutation must say what its clause does"

    unevidenced = [m.guard for m in MUTATIONS if m.granularity == Mutation.DETECTOR and not m.survived_clause]
    assert not unevidenced, (
        f"these entries claim DETECTOR granularity while naming no finer mutation that was tried: "
        f"{unevidenced}. Carry the survivor, or declare the finer clause instead."
    )
    misplaced = [m.guard for m in MUTATIONS if m.granularity == Mutation.CLAUSE and m.survived_clause]
    assert not misplaced, (
        f"these entries are graded CLAUSE but carry a surviving finer mutation: {misplaced}. If a "
        "finer clause exists and survives, the grade is DETECTOR."
    )
    half = [m.guard for m in MUTATIONS if bool(m.survived_clause) != bool(m.survived_replacement)]
    assert not half, f"a survivor needs both a clause and a replacement: {half}"


def test_every_entry_declares_a_SPECIFIC_diagnostic_it_must_redden_with():
    """`expect` is what stops exit code 1 from being the whole verdict, so it cannot be a token.

    An empty or near-empty expectation matches any output and hands the verdict straight back to the
    exit status — the state this field exists to leave. It must also be a substring of no other
    entry's, or two rows could be satisfied by one another's diagnostic.
    """
    vague = [(m.guard, m.expect) for m in MUTATIONS if len(m.expect.strip()) < 20]
    assert not vague, f"these expectations are too weak to distinguish one red from another: {vague}"
    for m in MUTATIONS:
        clashes = [o.guard for o in MUTATIONS if o is not m and m.expect in o.expect]
        assert not clashes, f"{m.guard}'s expectation is contained in {clashes}'s — neither is specific"


# ------------------------------------------------------------------------------------------------
# THE EXECUTION — one isolated repository, reused, reset between mutations
# ------------------------------------------------------------------------------------------------


@pytest.fixture(scope="session")
def sandbox(tmp_path_factory):
    """An isolated copy of this repository, with the POSITIVE CONTROL already run in it.

    The control is inside the fixture rather than in a test of its own so ordering is a dependency
    rather than a convention: no mutation can be judged before every proof test has been shown green
    on the unmutated tree. Without it, "the named test went red" is satisfied just as well by a proof
    test that was already broken, and reporting that as a verified mutation is the failure mode this
    whole file exists to remove.
    """
    sb = build_sandbox(tmp_path_factory.mktemp("mutation-sandbox"))
    # The sandbox holds what GIT TRACKS, so a proof test in a file that has never been `git add`ed is
    # simply absent there and the run below reports "file or directory not found" — accurate, and
    # unreadable as a diagnosis. Named here instead, because this is what a developer adding a guard
    # hits first.
    missing = sorted({m.node_id.split("::")[0] for m in MUTATIONS if not (sb / m.node_id.split("::")[0]).is_file()})
    assert not missing, (
        f"these proof files are not in the sandbox: {missing}. The sandbox is derived from "
        "`git ls-files`, so an unstaged new file is not in it — `git add` it and re-run."
    )
    result = run_pytest(sb, [m.node_id for m in MUTATIONS])
    assert result.returncode == 0, (
        "the proof tests named by the inventory are NOT green on an unmutated copy of this "
        "repository, so nothing below can distinguish 'the mutation was noticed' from 'the test was "
        f"already red'. Fix them first.\n{result.stdout[-4000:]}{result.stderr[-2000:]}"
    )
    assert "passed" in result.stdout, f"the control run collected nothing: {result.stdout!r}"
    try:
        yield sb
    finally:
        # ~115 MiB of tracked files plus its own git objects. pytest keeps the last three sessions'
        # tmp dirs by default, so leaving it costs a third of a gigabyte on a developer machine that
        # runs this a few times.
        shutil.rmtree(sb, ignore_errors=True)


def test_the_reset_restores_the_BASELINE_even_after_a_proof_COMMITS(sandbox):
    """One sandbox serves every mutation, so the reset has to be a reset to a fixed point.

    `git reset --hard` with no argument resets to whatever HEAD currently is. A proof test that
    commits inside the sandbox — several drive `git commit` for real — moves HEAD onto a commit
    carrying whatever was in the tree at the time, and every later reset would faithfully restore
    THAT. The contamination surfaces as an unrelated red several mutations further on, which is the
    hardest kind of harness defect to attribute.
    """
    subject = sandbox / "docs" / "guard-inventory.md"
    baseline = subject.read_text()

    subject.write_text(baseline + "\n<!-- planted, then COMMITTED -->\n")
    _git(sandbox, "add", "-A")
    _git(sandbox, "commit", "-qm", "a proof test committing inside the sandbox")
    assert subject.read_text() != baseline, "the planted change did not land, so this proves nothing"

    reset_sandbox(sandbox)

    assert subject.read_text() == baseline, (
        "the reset restored the sandbox to a commit made DURING a mutation rather than to the "
        "pristine baseline, so every later verdict is computed against a contaminated tree"
    )
    # Through `_git`, not a raw `subprocess.run`: an ambient `GIT_DIR` — which a git hook exports,
    # and this suite runs from one — would resolve this against the REAL repository and compare two
    # commits that have nothing to do with the sandbox. Asserting isolation with an unisolated call
    # is the defect this harness exists to catch, one level up.
    head = _git(sandbox, "rev-parse", "HEAD").stdout.decode().strip()
    assert head == _BASELINES[str(sandbox.resolve())], f"HEAD was left off the recorded baseline commit: {head}"


@pytest.mark.parametrize("mutation", MUTATIONS, ids=lambda m: m.guard)
def test_MUTATION_the_declared_clause_reddens_the_named_proof(sandbox, mutation):
    reset_sandbox(sandbox)
    verdict = verify_mutation(sandbox, mutation)
    assert verdict.ok, f"{mutation.guard}: {verdict.reason}"


_SURVIVORS = tuple(m for m in MUTATIONS if m.granularity == Mutation.DETECTOR)


@pytest.mark.parametrize("mutation", _SURVIVORS, ids=lambda m: m.guard)
def test_every_SURVIVING_clause_mutation_still_does(sandbox, mutation):
    """The DETECTOR grade, executed rather than recited.

    A finer mutation that has since STARTED reddening the proof test means the guard now admits
    clause-level proof and the entry should be regraded — the coarse grade would otherwise persist as
    an excuse long after the reason for it went away. Failing here is therefore good news; it just
    has to be acted on.
    """
    reset_sandbox(sandbox)
    finer = replace(
        mutation,
        clause=mutation.survived_clause,
        replacement=mutation.survived_replacement,
        survived_clause="",
        survived_replacement="",
    )
    verdict = verify_mutation(sandbox, finer)
    assert not verdict.ok, (
        f"{mutation.guard} is graded DETECTOR because {mutation.survived_clause!r} was tried and left "
        "the proof test green — but it reddens it now. Regrade the entry to CLAUSE with that mutation."
    )
    assert "still PASSED" in verdict.reason, (
        f"the survivor did not survive for the recorded reason — it failed with: {verdict.reason}"
    )


# ------------------------------------------------------------------------------------------------
# THIS GUARD'S OWN MUTATION PROOF — the redness clause, disarmed, on a sandbox of two files
# ------------------------------------------------------------------------------------------------

EXIT_STATUS_CLAUSE = "    if result.returncode != 1:"
DIAGNOSTIC_CLAUSE = "    if mutation.expect not in diagnostic:"
LIB = REPO_ROOT / "scripts" / "tests" / "mutation_harness_lib.py"


def _inert_sandbox(tmp_path: Path, body: str = "    assert True\n") -> tuple[Path, Mutation]:
    """A minimal synthetic sandbox holding one test file, whose outcome is fixed by `body`.

    Deliberately not a copy of the repo: the subject here is `verify_mutation`'s verdict, and a real
    sandbox would cost four seconds to prove something about two lines of control flow.
    """
    tests = tmp_path / "scripts" / "tests"
    tests.mkdir(parents=True)
    (tests / "test_inert.py").write_text(f"# INERT MARKER\n\n\ndef test_ok():\n{body}")
    return tmp_path, Mutation(
        guard="inert",
        target="scripts/tests/test_inert.py",
        clause="# INERT MARKER",
        replacement="# INERT MARKER, CHANGED",
        proof="test_inert.py::test_ok",
        granularity=Mutation.CLAUSE,
        # A legal expectation, not the empty string: an empty one is a shape the manifest forbids,
        # so a fixture relying on it would be proving something about a configuration that cannot
        # ship. Each caller that needs a different one passes it through `replace`.
        expect="a diagnostic no run of this fixture produces",
        why="an inert mutation: the named test cannot notice it",
    )


def _lib_with(tmp_path: Path, clause: str, replacement: str, label: str):
    """Import a copy of the library with one clause replaced.

    The copy is registered in `sys.modules` before execution: `@dataclass` resolves a string
    annotation through `sys.modules[cls.__module__]`, so an unregistered module raises AttributeError
    on the first dataclass it defines rather than on anything to do with the clause.
    """
    source = LIB.read_text()
    assert source.count(clause) == 1, (
        f"the {label} has moved or been reworded; RETARGET this mutation rather than loosening the "
        "match — and update the mutation_manifest entry for this file, which names the same string"
    )
    path = tmp_path / f"mutant_{label.replace(' ', '_')}.py"
    path.write_text(source.replace(clause, replacement, 1))

    spec = importlib.util.spec_from_file_location(path.stem, path)
    mutant = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = mutant
    try:
        spec.loader.exec_module(mutant)
    finally:
        sys.modules.pop(spec.name, None)
    return mutant


def test_an_INERT_mutation_is_REPORTED_rather_than_passed(tmp_path):
    """The behavioural half. A harness that cannot tell a real disarm from a comment edit would
    report every row verified on any tree at all."""
    sb, inert = _inert_sandbox(tmp_path)
    verdict = verify_mutation(sb, inert)
    assert not verdict.ok, "a mutation the named test cannot possibly notice was reported as verified"
    assert "still PASSED" in verdict.reason, verdict.reason


def test_a_red_for_the_WRONG_REASON_is_not_accepted(tmp_path):
    """The gate `expect` exists for. Pytest reports an ordinary exception exactly as it reports a
    failed assertion, so a mutation that CRASHES the proof test looks identical to one it detected.
    A verdict that cannot tell those apart certifies rows on evidence about nothing."""
    sb, inert = _inert_sandbox(tmp_path, "    raise RuntimeError('an unrelated crash')\n")
    crashing = replace(inert, expect="the diagnostic this row is supposed to produce")
    verdict = verify_mutation(sb, crashing)
    assert not verdict.ok, "a red with nothing to do with the declared diagnostic was accepted"
    assert "NOT with the declared diagnostic" in verdict.reason, verdict.reason


def test_MUTATION_disarming_the_DIAGNOSTIC_gate_accepts_a_red_for_the_wrong_reason(tmp_path):
    """The clause-level proof, on the newer of the two gates every verdict passes through.

    Disarm the check that the failure carries the declared diagnostic, and the crashing case above
    must start reporting as verified. If it does not, that rejection is coming from somewhere other
    than the clause, and the test above proves nothing about it.
    """
    sb, inert = _inert_sandbox(tmp_path, "    raise RuntimeError('an unrelated crash')\n")
    crashing = replace(inert, expect="the diagnostic this row is supposed to produce")
    assert not verify_mutation(sb, crashing).ok, (
        "the UNMUTATED verdict already accepted it, so the mutant proves nothing"
    )

    mutant = _lib_with(tmp_path, DIAGNOSTIC_CLAUSE, "    if False:", "diagnostic gate")
    assert mutant.verify_mutation(sb, crashing).ok, (
        "the diagnostic gate was replaced with a constant and a red for an unrelated reason was "
        "STILL rejected, so the verdict does not hang on the clause that reads it"
    )


def test_MUTATION_disarming_the_EXIT_STATUS_gate_accepts_a_run_that_NEVER_RAN_A_TEST(tmp_path):
    """The same proof for the older gate, and it has to be built carefully to isolate it.

    A GREEN run cannot serve: it produces no exception output, so only an EMPTY expectation would
    reach the status gate — and an empty expectation is a shape the manifest forbids, which would
    make this a proof about a configuration that cannot ship. Instead the sandbox's test file fails
    at IMPORT: pytest exits non-1 (nothing was collected, so nothing ran) while still printing the
    exception, so a legal non-empty expectation matches and the exit status is the ONLY thing
    rejecting it. That is the case this gate exists for — a proof ref that no longer names a
    collectable test must not read as a guard going red.
    """
    boom = "a deliberate import-time failure, which is not a test result"
    sb, inert = _inert_sandbox(tmp_path)
    (sb / "scripts" / "tests" / "test_inert.py").write_text(f"# INERT MARKER\nraise RuntimeError({boom!r})\n")
    uncollectable = replace(inert, expect=boom)

    verdict = verify_mutation(sb, uncollectable)
    assert not verdict.ok, "a run in which no test executed was accepted as a guard going red"
    assert len(uncollectable.expect) >= 20, "the expectation must be one the manifest would accept"

    mutant = _lib_with(tmp_path, EXIT_STATUS_CLAUSE, "    if False:", "exit status gate")
    assert mutant.verify_mutation(sb, uncollectable).ok, (
        "the exit-status gate was replaced with a constant and a run that never executed a test was "
        "still rejected, so the verdict does not hang on the status it reads"
    )


def test_a_clause_that_no_longer_OCCURS_ONCE_is_reported_rather_than_applied(tmp_path):
    """The failure paths, driven directly, because this is where a harness quietly stops harnessing.

    A clause that has been reworded away, or that now matches a second site, must produce a verdict
    naming the problem. Silently replacing nothing — or replacing the wrong site — would leave every
    row in this file reporting verified while mutating something nobody declared.
    """
    sb, inert = _inert_sandbox(tmp_path)

    gone = replace(inert, clause="# A CLAUSE THAT IS NOT THERE")
    assert not verify_mutation(sb, gone).ok
    assert "occurs 0 times" in verify_mutation(sb, gone).reason

    (sb / "scripts" / "tests" / "test_inert.py").write_text(
        "# INERT MARKER\n# INERT MARKER\n\n\ndef test_ok():\n    assert True\n"
    )
    assert "occurs 2 times" in verify_mutation(sb, inert).reason

    absent = replace(inert, target="scripts/tests/no_such_file.py")
    assert "does not exist in the sandbox" in verify_mutation(sb, absent).reason


def test_the_sandbox_is_left_UNCHANGED_by_a_verdict(tmp_path):
    """One sandbox serves every mutation, so a verdict that leaves its edit behind would make each
    result a function of the ones before it."""
    sb, inert = _inert_sandbox(tmp_path)
    subject = sb / "scripts" / "tests" / "test_inert.py"
    before = subject.read_text()
    verify_mutation(sb, inert)
    assert subject.read_text() == before, "verify_mutation left its mutation in the sandbox"