ersatztv/scripts/tests/test_check_required_contexts.py

"""Proof for `scripts/check-required-contexts.sh` (ersatztv#787).

The script reconciles the LIVE required status checks on `main` against the committed snapshot
`.gitea/required-status-contexts.json`, from which `test_ci_dropped_step_guard.py` derives its
marked-job scope. It is a pure classifier over a payload its caller fetched, which is the whole
reason it is a script at all rather than a few lines inside `pretooluse-merge-consent.sh`: the
merge hook's own comments record that while its verdict classification lived inline it had no tests
and three false-opens survived in it (#629).

Every test here execs the REAL script. Nothing reimplements its logic — a second copy of the
classifier in the test would agree with the first by construction and prove nothing.
"""

from __future__ import annotations

import json
import shutil
import subprocess
import tempfile
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / "scripts" / "check-required-contexts.sh"
SNAPSHOT = REPO_ROOT / ".gitea" / "required-status-contexts.json"

# THE CONTRACT, pinned. `pretooluse-merge-consent.sh` maps each of these onto a hook decision and
# treats anything else as "could not tell" -> ask. Pinning the set here means a sixth class cannot be
# added to the script without this test demanding the hook learn about it: an unmapped class would
# otherwise reach the hook's catch-all and read as a malfunction rather than the finding it is.
CLASSES = {"match", "drift", "nomatch", "undecidable", "unreadable"}

SNAP_CONTEXTS = json.loads(SNAPSHOT.read_text())["contexts"]


def run(payload: str, snapshot: Path | None = None, branch: str = "main") -> tuple[int, str, str]:
    proc = subprocess.run(
        [str(SCRIPT), "--branch", branch, "--snapshot", str(snapshot or SNAPSHOT)],
        input=payload,
        capture_output=True,
        text=True,
        check=False,
    )
    return proc.returncode, proc.stdout.strip(), proc.stderr


def rule(contexts, name="main", enabled=True):
    return {"branch_name": name, "enable_status_check": enabled, "status_check_contexts": contexts}


def test_the_script_is_executable():
    assert SCRIPT.is_file() and SCRIPT.stat().st_mode & 0o111, f"{SCRIPT} must exist and be executable"


def test_the_committed_snapshot_is_well_formed_and_not_empty():
    """ANTI-VACUITY. An empty snapshot would make `match` reachable against an empty live list while
    `test_ci_dropped_step_guard.py` derived an empty scope and collected zero parametrized cases."""
    data = json.loads(SNAPSHOT.read_text())
    assert isinstance(data.get("contexts"), list) and data["contexts"]
    assert all(isinstance(c, str) and c.strip() for c in data["contexts"])
    assert data.get("branch") == "main"
    assert data.get("read_on"), "the snapshot must date itself — it is a claim about a moment"


def test_live_equal_to_the_snapshot_is_match():
    assert run(json.dumps([rule(list(SNAP_CONTEXTS))]))[1] == "match"


def test_the_comparison_is_by_SET_not_by_order():
    """Gitea does not promise an order, so an order-sensitive compare would report drift forever —
    and a guard that cries wolf on every merge is one somebody switches off."""
    assert run(json.dumps([rule(list(reversed(SNAP_CONTEXTS)))]))[1] == "match"


def test_an_ADDED_required_context_is_drift():
    """THE FINDING #787 EXISTS FOR: a fourth required check whose job carries no execution marker."""
    code, out, _ = run(json.dumps([rule([*SNAP_CONTEXTS, "Build ErsatzTV Image / Fourth (pull_request)"])]))
    assert (code, out) == (0, "drift")


def test_a_REMOVED_required_context_is_drift():
    assert run(json.dumps([rule(list(SNAP_CONTEXTS[:-1]))]))[1] == "drift"


def test_null_contexts_against_a_non_empty_snapshot_is_drift():
    """`null` is a legitimate payload meaning "none required", not a read failure — so it is the
    FINDING that every required context was removed, never `unreadable`."""
    payload = json.dumps([{"branch_name": "main", "enable_status_check": True, "status_check_contexts": None}])
    assert run(payload)[1] == "drift"


def test_a_rule_with_status_checks_DISABLED_is_drift_against_a_nonempty_snapshot():
    """The EFFECTIVE required set, not the raw list.

    `enable_status_check: false` means Gitea requires nothing on the branch whatever
    `status_check_contexts` still holds. Type-checking the flag and ignoring its VALUE certifies
    disabled protection as a current mirror — measured: with that clause removed the same payload
    returns `match`, and all 71 tests across this file and the merge-consent file stayed green,
    because `rule()`'s `enabled=` parameter had no call site that ever passed False.
    """
    assert run(json.dumps([rule(list(SNAP_CONTEXTS), enabled=False)]))[1] == "drift"


def test_a_rule_with_status_checks_DISABLED_matches_an_EMPTY_snapshot(tmp_path):
    """The other side of the same clause: disabled protection IS an accurate mirror of a snapshot
    that requires nothing, so it must not be reported as drift."""
    empty = tmp_path / "snap.json"
    empty.write_text(json.dumps({"repo": "timothy/ersatztv", "branch": "main", "contexts": []}))
    assert run(json.dumps([rule(list(SNAP_CONTEXTS), enabled=False)]), snapshot=empty)[1] == "match"


def test_no_rule_governing_the_branch_is_nomatch():
    assert run(json.dumps([rule([], name="develop")]))[1] == "nomatch"


def test_a_glob_rule_makes_it_undecidable_even_when_an_exact_rule_matches():
    """ORDER MATTERS, and this is the test that pins it. Gitea picks the governing rule by Priority
    with gobwas/glob semantics, so a glob rule can outrank an exactly-named one. Preferring the exact
    rule would compare against a rule Gitea may not be applying and could report `match` on a branch
    whose real required set has drifted — a false-open in the direction that costs something."""
    payload = json.dumps([rule([], name="m*"), rule(list(SNAP_CONTEXTS))])
    assert run(payload)[1] == "undecidable"


def test_two_rules_folding_to_the_same_name_are_undecidable():
    assert run(json.dumps([rule(list(SNAP_CONTEXTS)), rule(list(SNAP_CONTEXTS), name="MAIN")]))[1] == "undecidable"


@pytest.mark.parametrize(
    ("label", "payload"),
    [
        ("not json", "not json at all"),
        ("top level object", '{"branch_name": "main"}'),
        (
            "enable_status_check as a string",
            json.dumps([{"branch_name": "main", "enable_status_check": "true", "status_check_contexts": []}]),
        ),
        ("a non-string context member", json.dumps([rule(["ok", 123])])),
        ("contexts as an object", json.dumps([rule({"a": 1})])),
    ],
)
def test_shapes_this_decision_cannot_consume_are_unreadable(label, payload):
    """Validated to the depth it is CONSUMED at, not just the top-level type. An object-typed check
    that never inspects its members is the one-level-down swallow the merge hook fixed twice."""
    assert run(payload)[1] == "unreadable", label


def test_every_class_the_script_emits_is_one_the_contract_declares():
    """Every class the contract DECLARES is reachable, and these five payloads produce no other.

    Deliberately not claimed: that a sixth class cannot ship. This drives five canned inputs, so a
    class emitted only on a sixth input shape would leave `seen` unchanged and pass — the hook's
    catch-all arm is what keeps that fail-closed, not this test. What it does catch is a declared
    class going unreachable (dead contract) or one of these five silently changing.
    """
    seen = {
        run(json.dumps([rule(list(SNAP_CONTEXTS))]))[1],
        run(json.dumps([rule([*SNAP_CONTEXTS, "x (pull_request)"])]))[1],
        run(json.dumps([rule([], name="develop")]))[1],
        run(json.dumps([rule([], name="m*")]))[1],
        run("not json")[1],
    }
    assert seen == CLASSES, f"classes reached {sorted(seen)} != contract {sorted(CLASSES)}"


def test_a_usage_error_exits_2_and_prints_NO_class():
    """A broken invocation must not be mistakable for a finding. If it printed a class AND exited
    non-zero, a caller reading only stdout would act on a comparison that never happened."""
    proc = subprocess.run([str(SCRIPT), "--bogus"], input="[]", capture_output=True, text=True, check=False)
    assert proc.returncode == 2
    assert proc.stdout.strip() == ""
    assert not (CLASSES & set(proc.stdout.split()))


def test_an_unreadable_snapshot_is_a_usage_error_not_a_class():
    proc = subprocess.run(
        [str(SCRIPT), "--snapshot", str(REPO_ROOT / "does" / "not" / "exist.json")],
        input="[]",
        capture_output=True,
        text=True,
        check=False,
    )
    assert proc.returncode == 2 and proc.stdout.strip() == ""


def test_a_snapshot_naming_NO_branch_is_a_usage_error(tmp_path):
    """The snapshot must name the branch it mirrors. Treating an absent `branch` as "compare against
    whatever was asked for" reaches the same defect as a mismatched one — one branch's live contexts
    weighed against another's mirror — by omission rather than by disagreement."""
    bad = tmp_path / "snap.json"
    bad.write_text(json.dumps({"contexts": list(SNAP_CONTEXTS)}))
    code, out, err = run(json.dumps([rule(list(SNAP_CONTEXTS))]), snapshot=bad)
    assert (code, out) == (2, ""), "an unbranded snapshot must be a usage error, never a class"
    # THE ARM-UNIQUE WORDING. Exit 2 alone cannot see this clause: with `.branch` absent
    # `snap_branch` is empty, and `--branch` can never be empty, so the MISMATCH guard on the next
    # line fires anyway with the same exit code and the same empty stdout. Two guards masking each
    # other (#685) — measured: deleting the required-branch clause left this file at 27 passed.
    assert "names no branch" in err, (
        f"the required-branch clause did not fire; the mismatch guard answered instead: {err.strip()[:160]}"
    )


def test_a_snapshot_for_a_DIFFERENT_branch_is_a_usage_error():
    code, out, _ = run(json.dumps([rule(list(SNAP_CONTEXTS))]), branch="develop")
    assert (code, out) == (2, ""), "a branch mismatch must be a usage error, never a class"


def test_a_malformed_snapshot_is_unreadable_not_a_confident_drift(tmp_path):
    """The snapshot is validated to the same depth as the live payload. A `contexts` holding a
    non-string would otherwise compare unequal against a well-formed live list and be reported as
    `drift` — a confident finding derived from data that was never understood."""
    bad = tmp_path / "snap.json"
    bad.write_text(json.dumps({"contexts": ["ok", 7]}))
    assert run(json.dumps([rule(list(SNAP_CONTEXTS))]), snapshot=bad)[1] == "unreadable"


# The clause this guard hangs on: the set comparison that turns a differing live list into `drift`.
COMPARE_CLAUSE = "($rule.status_check_contexts | sort | unique) == $snap"


def test_a_glob_that_could_NOT_govern_the_branch_does_not_block_the_comparison():
    """Rule selection is delegated to `scripts/lib/branch-rule-classifier.jq`, the SAME program
    `pretooluse-merge-consent.sh` loads. A rule named `a[b` carries a metacharacter but cannot govern
    `main`, so it must not make the comparison undecidable — a blanket "any metacharacter anywhere"
    test would ask forever in a repo that has one, which is how a correct guard gets switched off."""
    payload = json.dumps([rule([], name="a[b"), rule(list(SNAP_CONTEXTS))])
    assert run(payload)[1] == "match"


CLASSIFIER = REPO_ROOT / "scripts" / "lib" / "branch-rule-classifier.jq"
MERGE_HOOK = REPO_ROOT / ".claude" / "hooks" / "pretooluse-merge-consent.sh"


SENTINEL_CLASSIFIER = (
    "# sentinel: a classifier that answers only one way, so a caller that loads THIS file changes\n"
    '{verdict: "undecidable"}\n'
)


def test_ONE_classifier_TWO_callers_BEHAVIOURALLY():
    """The structural property #787 relies on, PROVEN by swapping the file — not by reading source.

    A string-presence check was the seventh test in this change to pass for the wrong reason: cold
    review re-inlined a BYTE-IDENTICAL copy of the classifier into the merge hook, left a comment
    naming `scripts/lib/branch-rule-classifier.jq` above it, and the whole suite stayed green at
    exactly 1062 passed. A comment satisfies a substring assertion. And the byte-identical inline is
    precisely the drift-creating refactor that matters, because it AGREES on day one — a weakened
    copy would have been caught behaviourally.

    So the proof replaces the shared file with a sentinel that can only answer `undecidable`, and
    requires the caller's behaviour to change. Nothing but actually loading that path can do that.
    """
    assert CLASSIFIER.is_file(), f"the shared classifier is missing at {CLASSIFIER}"
    with tempfile.TemporaryDirectory() as tmp:
        root = Path(tmp)
        victim = root / "check-required-contexts.sh"
        victim.write_text(SCRIPT.read_text(encoding="utf-8"))
        victim.chmod(0o755)
        (root / "lib").mkdir()
        (root / "lib" / "branch-rule-classifier.jq").write_text(SENTINEL_CLASSIFIER)

        payload = json.dumps([rule(list(SNAP_CONTEXTS))])
        proc = subprocess.run(
            [str(victim), "--branch", "main", "--snapshot", str(SNAPSHOT)],
            input=payload,
            capture_output=True,
            text=True,
            check=False,
        )
    assert (proc.returncode, proc.stdout.strip()) == (0, "undecidable"), (
        "swapping scripts/lib/branch-rule-classifier.jq did not change this script's verdict "
        f"(exit={proc.returncode} stdout={proc.stdout.strip()!r}) — it is not loading that file, so "
        "the one-copy property is a claim rather than a fact"
    )


def _mutant_tree(tmp_path, source_text):
    """The script in a RUNNABLE layout: it resolves `lib/branch-rule-classifier.jq` relative to its
    own directory, so a bare copy exits 2 before reaching any of its logic.

    This is not incidental plumbing. The first version of the mutation proof below copied only the
    script, so the mutant died on `classifier not readable` with EMPTY stdout — and an assertion that
    stdout merely differed from `drift` was satisfied by that empty string. The proof passed for the
    wrong reason and the `MUTATION` grade it justified was unsupported.
    """
    victim = tmp_path / "mutant.sh"
    victim.write_text(source_text)
    victim.chmod(0o755)
    shutil.copytree(REPO_ROOT / "scripts" / "lib", tmp_path / "lib")
    return victim


def test_the_mutation_harness_layout_ITSELF_reproduces_the_real_verdict(tmp_path):
    """POSITIVE CONTROL for the harness below, without which it can only prove a broken copy.

    The UNMUTATED script, in the same tmp layout, must reach the same verdict it reaches in the
    repo. If this fails, the mutation test's red says nothing about the clause.
    """
    victim = _mutant_tree(tmp_path, SCRIPT.read_text(encoding="utf-8"))
    drifted = json.dumps([rule([*SNAP_CONTEXTS, "Build ErsatzTV Image / Fourth (pull_request)"])])
    proc = subprocess.run(
        [str(victim), "--branch", "main", "--snapshot", str(SNAPSHOT)],
        input=drifted,
        capture_output=True,
        text=True,
        check=False,
    )
    assert (proc.returncode, proc.stdout.strip()) == (0, "drift"), (
        f"the unmutated script did not report drift in the harness layout "
        f"(exit={proc.returncode} stdout={proc.stdout.strip()!r} stderr={proc.stderr.strip()[:160]!r}) — "
        "the mutation proof below would then be measuring a broken copy, not a disarmed clause"
    )


def test_a_MISSING_shared_classifier_is_a_usage_error_not_an_unreadable_CLASS():
    """The twin of a fix the hook made loudly, on the second caller.

    Without this guard `jq -f` on a missing program fails, `verdict` is empty, and the script prints
    the CLASS `unreadable` with exit 0 — reporting "the payload came back in a shape I could not
    consume" about a missing LOCAL FILE. That is the states-a-cause-that-did-not-happen defect the
    hook added its own `[ -r "$classifier" ]` check to avoid; measured, removing this one left the
    file at 27 passed.
    """
    with tempfile.TemporaryDirectory() as tmp:
        victim = Path(tmp) / "check-required-contexts.sh"
        victim.write_text(SCRIPT.read_text(encoding="utf-8"))
        victim.chmod(0o755)
        # deliberately NO lib/ beside it
        proc = subprocess.run(
            [str(victim), "--branch", "main", "--snapshot", str(SNAPSHOT)],
            input=json.dumps([rule(list(SNAP_CONTEXTS))]),
            capture_output=True,
            text=True,
            check=False,
        )
    assert proc.returncode == 2, f"a missing classifier must be a usage error, got exit {proc.returncode}"
    assert proc.stdout.strip() == "", (
        f"a missing local file was reported as the class {proc.stdout.strip()!r} — a cause that did not happen"
    )
    assert "classifier not readable" in proc.stderr


def test_MUTATION_disarming_the_set_comparison_stops_every_drift_report(tmp_path):
    """`testing.guard-ships-with-mutation-proof` (#775) — the DECLARED clause, re-run every suite.

    The set comparison is the guard's entire finding. Disarm that clause alone — make it constantly
    true — and a live list that genuinely differs from the snapshot must stop reporting `drift`. If
    it still did, the finding comes from somewhere other than the clause this guard hangs on and
    every passing test above proves nothing about it.
    """
    text = SCRIPT.read_text(encoding="utf-8")
    assert COMPARE_CLAUSE in text, (
        "the set-comparison clause has moved or been reworded; RETARGET this mutation at its new "
        "location rather than loosening the string match — a mutation that silently stops mutating "
        "is the exact failure this file exists to catch"
    )
    mutated = text.replace(COMPARE_CLAUSE, "true", 1)
    assert mutated != text and COMPARE_CLAUSE not in mutated, "the replacement did not change the source"

    victim = _mutant_tree(tmp_path, mutated)
    drifted = json.dumps([rule([*SNAP_CONTEXTS, "Build ErsatzTV Image / Fourth (pull_request)"])])
    proc = subprocess.run(
        [str(victim), "--branch", "main", "--snapshot", str(SNAPSHOT)],
        input=drifted,
        capture_output=True,
        text=True,
        check=False,
    )
    # EXACT, not `!= "drift"`. The mutant must RUN and reach the disarmed clause, so the only
    # acceptable evidence is the specific wrong answer it now gives. Any exit code but 0, or an
    # empty stdout, means it died on the way and proves nothing.
    assert (proc.returncode, proc.stdout.strip()) == (0, "match"), (
        f"the mutant did not reach the disarmed clause and report `match` "
        f"(exit={proc.returncode} stdout={proc.stdout.strip()!r} stderr={proc.stderr.strip()[:160]!r}). "
        "With the set comparison constantly true it must certify a drifted list as current; "
        "anything else means the proof measured a broken copy rather than a disarmed guard"
    )