ersatztv/scripts/tests/test_prove_fix.py

"""scripts/prove-fix.sh witnesses a fix's test failing before the fix (ersatztv#794).

THE NEGATIVE CONTROL IS THE POINT OF THIS FILE. A prover that reports PROVEN for
everything is worse than no prover: it manufactures exactly the confidence #794 exists
to withhold. So `test_unrelated_test_is_UNPROVEN` is the load-bearing case here, and the
positive case only tells us the script can distinguish the two.

Each test builds a throwaway git repo rather than pinning real commits from this
repository's history — a test anchored to a real sha rots the moment that sha is rebased
or the file moves, and then it passes for the wrong reason (or is deleted for being
flaky, which is worse).
"""

from __future__ import annotations

import os
import subprocess
import time
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parents[2]
# Overridable so a MUTANT copy can be driven through these very tests — that is what makes
# the mutation proof below a witnessed red rather than an argument. Gated behind a sentinel
# so a stray CI value cannot silently point the whole suite at another script.
if os.environ.get("PROVE_FIX_PATH") and os.environ.get("PROVE_FIX_MUTATION_RUN") != "1":
    raise RuntimeError(
        "PROVE_FIX_PATH is set without PROVE_FIX_MUTATION_RUN=1. That would silently test a "
        "different script than the one this suite vouches for."
    )
PROVE_FIX = Path(os.environ.get("PROVE_FIX_PATH") or (REPO_ROOT / "scripts" / "prove-fix.sh"))


def _git(repo: Path, *args: str) -> str:
    return subprocess.run(
        ["git", "-C", str(repo), *args],
        check=True,
        capture_output=True,
        text=True,
    ).stdout.strip()


def _run(repo: Path, *args: str) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        ["bash", str(PROVE_FIX), "--repo", str(repo), *args],
        capture_output=True,
        text=True,
        cwd=str(repo),
    )


@pytest.fixture
def fixrepo(tmp_path: Path) -> Path:
    """A repo whose HEAD is a fix: code change + a test that discriminates.

    It also carries an UNRELATED test, present from the first commit, which passes with
    or without the fix. That test is the negative control's subject.
    """
    repo = tmp_path / "r"
    (repo / "scripts" / "tests").mkdir(parents=True)
    subprocess.run(["git", "init", "-q", str(repo)], check=True)
    _git(repo, "config", "user.email", "t@example.com")
    _git(repo, "config", "user.name", "T")

    # --- commit 1: the bug, plus a test that cannot see it
    (repo / "calc.py").write_text("def add(a, b):\n    return a - b  # bug\n")
    (repo / "scripts" / "tests" / "test_unrelated.py").write_text("def test_unrelated():\n    assert 1 + 1 == 2\n")
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "initial: buggy add, unrelated test")

    # --- commit 2: the fix + a test that discriminates against its absence
    (repo / "calc.py").write_text("def add(a, b):\n    return a + b\n")
    (repo / "scripts" / "tests" / "test_add.py").write_text(
        "from calc import add\n\n\ndef test_add():\n    assert add(2, 3) == 5\n"
    )
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "fix: add() returned a difference\n\nProves: scripts/tests/test_add.py")
    return repo


def test_reverting_the_fix_reddens_its_test_PROVEN(fixrepo: Path) -> None:
    r = _run(fixrepo, "HEAD", "scripts/tests/test_add.py")
    assert r.returncode == 0, f"expected PROVEN (0), got {r.returncode}\n{r.stdout}\n{r.stderr}"
    assert "PROVEN" in r.stdout
    assert "UNPROVEN" not in r.stdout


def test_unrelated_test_is_UNPROVEN(fixrepo: Path) -> None:
    """THE NEGATIVE CONTROL. A test that passes without the fix must be refused.

    Without this, every other assertion in this file is compatible with a script that
    prints PROVEN unconditionally.
    """
    r = _run(fixrepo, "HEAD", "scripts/tests/test_unrelated.py")
    assert r.returncode == 1, f"expected UNPROVEN (1), got {r.returncode}\n{r.stdout}\n{r.stderr}"
    assert "UNPROVEN" in r.stdout
    assert "do not discriminate" in r.stdout


def test_selector_comes_from_the_Proves_trailer(fixrepo: Path) -> None:
    """No selector argument: it must read `Proves:` rather than guess."""
    r = _run(fixrepo, "HEAD")
    assert r.returncode == 0, f"expected PROVEN via trailer, got {r.returncode}\n{r.stdout}\n{r.stderr}"
    assert "scripts/tests/test_add.py" in r.stdout


def test_no_selector_and_no_trailer_REFUSES(fixrepo: Path) -> None:
    """Refusing beats guessing: a heuristic silently proves nothing when a fix edits an
    existing test, which is the failure mode being mechanised against."""
    (fixrepo / "calc.py").write_text("def add(a, b):\n    return a + b  # touched\n")
    _git(fixrepo, "add", "-A")
    _git(fixrepo, "commit", "-q", "-m", "fix: no trailer here")
    r = _run(fixrepo, "HEAD")
    assert r.returncode == 3, f"expected 3 (no selector), got {r.returncode}\n{r.stderr}"
    assert "Proves:" in r.stderr


def test_test_only_commit_REFUSES(fixrepo: Path) -> None:
    """A commit with no code side cannot be proven this way — it must say so, not pass."""
    (fixrepo / "scripts" / "tests" / "test_extra.py").write_text("def test_x():\n    assert True\n")
    _git(fixrepo, "add", "-A")
    _git(fixrepo, "commit", "-q", "-m", "test: add a test only\n\nProves: scripts/tests/test_extra.py")
    r = _run(fixrepo, "HEAD")
    assert r.returncode == 4, f"expected 4 (nothing to revert), got {r.returncode}\n{r.stderr}"
    assert "nothing to revert" in r.stderr


def test_selector_matching_no_tests_REFUSES(fixrepo: Path) -> None:
    """`pytest` exits 5 when it collects nothing. Treating that as red would prove every
    fix — a check that examined nothing reporting success."""
    r = _run(fixrepo, "HEAD", "scripts/tests/test_does_not_exist.py")
    assert r.returncode == 5, f"expected 5 (no tests collected), got {r.returncode}\n{r.stderr}"
    assert "NO tests" in r.stderr


def test_added_code_file_is_removed_not_checked_out(tmp_path: Path) -> None:
    """A file the fix ADDED does not exist in the parent. `git checkout parent -- <new>`
    fails there, and if that failure were swallowed the fix would stay in place and every
    run would report a false PROVEN."""
    repo = tmp_path / "r2"
    (repo / "scripts" / "tests").mkdir(parents=True)
    subprocess.run(["git", "init", "-q", str(repo)], check=True)
    _git(repo, "config", "user.email", "t@example.com")
    _git(repo, "config", "user.name", "T")
    (repo / "scripts" / "tests" / "keep.py").write_text("# placeholder\n")
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "initial")

    # the fix ADDS helper.py (it has no parent version) and a test that needs it
    (repo / "helper.py").write_text("def shout(s):\n    return s.upper()\n")
    (repo / "scripts" / "tests" / "test_helper.py").write_text(
        "from helper import shout\n\n\ndef test_shout():\n    assert shout('a') == 'A'\n"
    )
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "feat: add shout\n\nProves: scripts/tests/test_helper.py")

    r = _run(repo, "HEAD")
    assert r.returncode == 0, f"expected PROVEN, got {r.returncode}\n{r.stdout}\n{r.stderr}"
    assert "PROVEN" in r.stdout


def test_root_commit_REFUSES(tmp_path: Path) -> None:
    repo = tmp_path / "r3"
    repo.mkdir()
    subprocess.run(["git", "init", "-q", str(repo)], check=True)
    _git(repo, "config", "user.email", "t@example.com")
    _git(repo, "config", "user.name", "T")
    (repo / "a.py").write_text("x = 1\n")
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "root")
    r = _run(repo, "HEAD", "scripts/tests")
    assert r.returncode == 5
    assert "root commit" in r.stderr


def test_runs_under_the_system_bash(fixrepo: Path) -> None:
    """macOS ships /bin/bash 3.2, where `mapfile` is absent AND yields an empty array
    instead of erroring. Assert by EXECUTING under that bash, not by grepping the source
    for "mapfile" — a grep matches the comment explaining why mapfile is avoided, which is
    a string predicate failing exactly as docs/defect-shapes-773.md §3.7 says they do."""
    system_bash = Path("/bin/bash")
    if not system_bash.exists():
        pytest.skip("/bin/bash not present")
    # Named for the system bash, NOT for 3.2: on the Linux runner /bin/bash is 5.x, so a
    # name promising bash-3.2 coverage would read as coverage that exists only on a
    # developer Mac. The 3.2 hazard (mapfile yielding an empty array) is what motivated it.
    ver = subprocess.run([str(system_bash), "--version"], capture_output=True, text=True).stdout
    r = subprocess.run(
        [str(system_bash), str(PROVE_FIX), "--repo", str(fixrepo), "HEAD", "scripts/tests/test_add.py"],
        capture_output=True,
        text=True,
        cwd=str(fixrepo),
    )
    assert r.returncode == 0, (
        f"prove-fix.sh must work under the system bash ({ver.splitlines()[0] if ver else '?'}); "
        f"got {r.returncode}\n{r.stdout}\n{r.stderr}"
    )
    assert "PROVEN" in r.stdout


def test_control_failure_REFUSES(fixrepo: Path) -> None:
    """A test that is ALREADY red with the fix in place proves nothing by being red after
    a revert. Without this control, half a discrimination claim reads as the whole one."""
    (fixrepo / "scripts" / "tests" / "test_broken.py").write_text(
        "from calc import add\n\n\ndef test_broken():\n    assert add(2, 3) == 99\n"
    )
    (fixrepo / "calc.py").write_text("def add(a, b):\n    return a + b  # unchanged\n")
    _git(fixrepo, "add", "-A")
    _git(fixrepo, "commit", "-q", "-m", "fix: with an already-failing test\n\nProves: scripts/tests/test_broken.py")
    r = _run(fixrepo, "HEAD")
    assert r.returncode == 6, f"expected 6 (control failed), got {r.returncode}\n{r.stdout}\n{r.stderr}"
    assert "control FAILED" in r.stderr


def test_control_green_then_red_is_reported_as_both(fixrepo: Path) -> None:
    """The PROVEN line must state both halves — green with, red without — because that
    pair is the claim. Reporting only the red half is the overclaim being mechanised out."""
    r = _run(fixrepo, "HEAD", "scripts/tests/test_add.py")
    assert r.returncode == 0
    assert "control" in r.stdout.lower()
    assert "Green with the fix (control), red without it" in r.stdout


def test_MUTATION_disarming_the_UNPROVEN_clause_reddens_the_refusal_test(
    tmp_path: Path,
) -> None:
    """Clause-level mutation, EXECUTED, with the unchanged test WITNESSED RED against it.

    `docs/guard-inventory.md` is explicit that MUTATION means a mutation was executed and
    the named test was *witnessed red* — feeding the real script a rejecting input is
    BEHAVIOUR-ONLY, and the column "is not a grading curve". Deleting the clause and then
    asserting the MUTANT misbehaved leaves this test green and proves nothing about
    whether the clause is load-bearing.

    So: delete the `RC -eq 0 -> UNPROVEN` clause (the CLAUSE, not the file — #510), then
    re-run the UNCHANGED `test_unrelated_test_is_UNPROVEN` against the mutant in a nested
    pytest run. That test must go RED. Its red is the proof.
    """
    real = REPO_ROOT / "scripts" / "prove-fix.sh"
    src = real.read_text()
    marker = 'if [ "$RC" -eq 0 ]; then'
    assert marker in src, "the clause under mutation is gone — regrade the inventory row"

    end = src.index("  exit 1\nfi\n", src.index(marker)) + len("  exit 1\nfi\n")
    mutant = tmp_path / "prove-fix-mutant.sh"
    mutant.write_text(src[: src.index(marker)] + src[end:])
    assert marker not in mutant.read_text(), "mutation did not remove the clause"

    env = {**os.environ, "PROVE_FIX_PATH": str(mutant), "PROVE_FIX_MUTATION_RUN": "1"}
    nested = subprocess.run(
        [
            "python3",
            "-m",
            "pytest",
            f"{Path(__file__).name}::test_unrelated_test_is_UNPROVEN",
            "-q",
            "-p",
            "no:cacheprovider",
        ],
        cwd=str(Path(__file__).parent),
        env=env,
        capture_output=True,
        text=True,
    )
    out = nested.stdout + nested.stderr
    assert nested.returncode == 1, (
        "the unchanged refusal test must go RED against the mutant (pytest exit 1). "
        f"got {nested.returncode} — exit 2/3/4/5 would mean the nested run broke rather "
        f"than the test failing, which proves nothing.\n{out[-2000:]}"
    )
    # Require a real reported FAILURE of that specific test. `returncode != 0` alone would
    # be satisfied by a collection error — the vacuous shape this whole file is against.
    assert "FAILED" in out and "test_unrelated_test_is_UNPROVEN" in out, (
        "expected a reported failure of test_unrelated_test_is_UNPROVEN; the nested run "
        f"failed for some other reason.\n{out[-2000:]}"
    )
    assert "1 failed" in out, f"expected exactly one failing test.\n{out[-1200:]}"


def test_SIGTERM_mid_run_never_reports_PROVEN(tmp_path: Path) -> None:
    """A killed run must not look like evidence.

    An early DRAFT printed PROVEN and exited 0 after a SIGTERM: the trap cleaned up but did
    not exit, so the previous status stood. On the first COMMITTED version (587edbecc) the
    run reaches rc 5 by a different route entirely, so this test earns its keep only via the
    assertions below — see the comment there.
    """
    repo = tmp_path / "slow"
    (repo / "scripts" / "tests").mkdir(parents=True)
    subprocess.run(["git", "init", "-q", str(repo)], check=True)
    _git(repo, "config", "user.email", "t@example.com")
    _git(repo, "config", "user.name", "T")
    (repo / "mod.py").write_text("VALUE = 1\n")
    (repo / "scripts" / "tests" / "test_slow.py").write_text(
        "import time\nfrom mod import VALUE\n\n\ndef test_slow():\n    time.sleep(20)\n    assert VALUE == 2\n"
    )
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "initial")
    (repo / "mod.py").write_text("VALUE = 2\n")
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "fix: bump\n\nProves: scripts/tests/test_slow.py")

    proc = subprocess.Popen(
        ["bash", str(PROVE_FIX), "--repo", str(repo), "HEAD"],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        cwd=str(repo),
    )
    time.sleep(4)  # inside the control run, which sleeps 20s
    assert proc.poll() is None, "the run finished before it could be signalled; test is void"
    proc.terminate()
    out, err = proc.communicate(timeout=60)

    # Assert the observable THIS fix introduced, not merely "non-zero and no PROVEN":
    # the pre-fix script also satisfied those two, by accident — its `trap cleanup EXIT INT
    # TERM` fired, deleted $TMP, execution continued, and a later step died 5. Two different
    # bugs landing on the same observable is not a witnessed fix. That pair was measured
    # failing to separate old from new; rc==5 AND the handler's own message do separate
    # them.
    assert proc.returncode == 5, f"a signalled run must exit 5 from on_signal, got {proc.returncode}\n{out}\n{err}"
    assert "interrupted by signal" in err, (
        f"expected the signal handler's own message, so this test cannot be satisfied by an "
        f"unrelated later failure:\n{err}"
    )
    assert "PROVEN" not in out, f"a signalled run must not print a verdict:\n{out}"


def test_a_harness_failure_is_NOT_reported_as_PROVEN(tmp_path: Path) -> None:
    """The marker-absence branch, reached the way the old false green was reached.

    Stubbing `python3` to exit 127 does NOT reach the marker logic: the marker IS written
    (127) and the script exits via the control-failure branch. Measured: a commit taking
    that route came back UNPROVEN from this tool's own gate, its fix executed by no test.

    The real shape is `( cd X && pytest ); rc=$?` returning 1 because `cd` FAILED and pytest
    never ran; pre-fix that was accepted as red and produced PROVEN. Reproduced by shimming
    `git` so the SECOND `worktree add` (the reverted phase) exits 0 without creating the
    directory. The fixture's fix ADDS its code file, so the revert step is `rm -f` — which
    succeeds on a missing directory and lets execution reach the phase's `cd`.
    """
    repo = tmp_path / "addrepo"
    (repo / "scripts" / "tests").mkdir(parents=True)
    subprocess.run(["git", "init", "-q", str(repo)], check=True)
    _git(repo, "config", "user.email", "t@example.com")
    _git(repo, "config", "user.name", "T")
    (repo / "scripts" / "tests" / "keep.py").write_text("# placeholder\n")
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "initial")
    (repo / "added.py").write_text("def val():\n    return 7\n")
    (repo / "scripts" / "tests" / "test_added.py").write_text(
        "from added import val\n\n\ndef test_val():\n    assert val() == 7\n"
    )
    _git(repo, "add", "-A")
    _git(repo, "commit", "-q", "-m", "feat: add val\n\nProves: scripts/tests/test_added.py")

    shim = tmp_path / "bin"
    shim.mkdir()
    counter = tmp_path / "count"
    (shim / "git").write_text(
        "#!/bin/sh\n"
        'if [ "$3" = "worktree" ] && [ "$4" = "add" ]; then\n'
        f'  n=$(cat "{counter}" 2>/dev/null || echo 0); n=$((n+1)); echo "$n" > "{counter}"\n'
        '  if [ "$n" -ge 2 ]; then exit 0; fi\n'
        "fi\n"
        'exec /usr/bin/git "$@"\n'
    )
    (shim / "git").chmod(0o755)

    env = {**os.environ, "PATH": f"{shim}:{os.environ['PATH']}"}
    r = subprocess.run(
        ["bash", str(PROVE_FIX), "--repo", str(repo), "HEAD"],
        capture_output=True,
        text=True,
        cwd=str(repo),
        env=env,
    )
    assert "PROVEN" not in r.stdout, (
        "a phase whose worktree does not exist cannot witness anything; pre-fix this "
        f"produced PROVEN from the subshell's status:\n{r.stdout}\n{r.stderr}"
    )
    assert r.returncode == 5, f"expected refusal (5), got {r.returncode}\n{r.stdout}\n{r.stderr}"
    assert "did not complete" in r.stderr, (
        f"expected the marker-absence diagnostic, not some other refusal:\n{r.stderr}"
    )