ersatztv/scripts/tests/test_build_catalog_check_path.py

"""`scripts/build_decisions_catalog.py`'s `--check` path — the one CI actually runs.

`decisions-guard` (`.gitea/workflows/pr-checks.yml`, the "Active catalog in sync" step) runs
`python3 scripts/build_decisions_catalog.py --check`. `scripts/tests/test_build_catalog.py` covers
`render_catalog()` directly and never calls `main()` at all, so nothing there proves:

  * that `main()`'s stale-detection comparison (`want.strip() != have.strip()`) is load-bearing —
    a version that always agreed would pass every existing test;
  * that the `if __name__ == "__main__": raise SystemExit(main())` wiring actually turns a stale
    catalog into a non-zero process exit code, which is the only thing CI's `run:` step reads.

`testing.guard-ships-with-mutation-proof` (#775) is explicit that a guard is not tested because a
test *involving* it passes: it ships with a mutation proof — disarm the guard's own clause, alone,
and a named test must go red. This file is that proof for the catalog-guard, plus the subprocess
proof that the `__main__` wiring is connected (the #751/#719 shape the decision record names: a
green `main()` behind dead wiring).
"""

from __future__ import annotations

import importlib.util
import re
import subprocess
import sys
from pathlib import Path

import pytest
import yaml

import scripts.build_decisions_catalog as bc
import scripts.decisions_lib as dl

REPO_ROOT = Path(__file__).resolve().parents[2]
CATALOG_SCRIPT = REPO_ROOT / "scripts" / "build_decisions_catalog.py"
DECISIONS_LIB = REPO_ROOT / "scripts" / "decisions_lib.py"
SCRIPTS_INIT = REPO_ROOT / "scripts" / "__init__.py"


def _current_catalog_text() -> str:
    return bc.render_catalog(dl.all_active_records())


# ------------------------------------------------------------------------------------------------
# ANTI-VACUITY — if the real corpus is empty, every assertion below passes for nothing.
# ------------------------------------------------------------------------------------------------


def test_the_real_corpus_is_non_empty_and_renders_a_real_catalog():
    records = dl.all_active_records()
    active = [r for r in records if r.status == "active" and r.key]
    assert active, "no active decision records were parsed — every test below would be vacuous"

    text = _current_catalog_text()
    assert bc.BANNER in text, "render_catalog produced no banner — not a real catalog document"
    assert f"`{active[0].key}`" in text, (
        "render_catalog produced no row for a known active record — not a real catalog document"
    )


# ------------------------------------------------------------------------------------------------
# `main(["--check"])` — the comparison CI reads
# ------------------------------------------------------------------------------------------------


def test_check_returns_0_when_OUTPUT_matches_render_catalog(tmp_path, monkeypatch):
    fresh = _current_catalog_text()
    output = tmp_path / "README.md"
    output.write_text(fresh.rstrip("\n") + "\n", encoding="utf-8")
    monkeypatch.setattr(bc, "OUTPUT", output)

    assert bc.main(["--check"]) == 0


def test_check_returns_1_when_OUTPUT_is_stale(tmp_path, monkeypatch):
    fresh = _current_catalog_text()
    output = tmp_path / "README.md"
    # Append a line: the on-disk file no longer matches what render_catalog would produce.
    output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")
    monkeypatch.setattr(bc, "OUTPUT", output)

    assert bc.main(["--check"]) == 1


def test_generate_then_check_round_trips(tmp_path, monkeypatch):
    """The no-argument path WRITES the catalog, and a subsequent --check must then pass.

    This pins the property CI depends on: generate and check agree. If they ever diverged, `main([])`
    would produce a file that `main(["--check"])` immediately rejects — a self-contradiction that
    would make the generator useless for fixing the exact problem `--check` reports.
    """
    output = tmp_path / "README.md"
    assert not output.exists()
    monkeypatch.setattr(bc, "OUTPUT", output)

    assert bc.main([]) == 0
    assert output.exists(), "main([]) with no --check must write OUTPUT"
    written = output.read_text(encoding="utf-8")
    assert written.strip() == _current_catalog_text().strip()

    assert bc.main(["--check"]) == 0, "the file main([]) just wrote must satisfy main(['--check'])"


# ------------------------------------------------------------------------------------------------
# THE `__main__` WIRING — proof CI's subprocess invocation actually surfaces staleness
# ------------------------------------------------------------------------------------------------


def _seed_decisions_copy(root: Path) -> None:
    """Copy only what build_decisions_catalog.py + decisions_lib.py need to resolve a real corpus."""
    (root / "scripts").mkdir(parents=True)
    (root / "scripts" / "__init__.py").write_bytes(SCRIPTS_INIT.read_bytes())
    (root / "scripts" / "build_decisions_catalog.py").write_bytes(CATALOG_SCRIPT.read_bytes())
    (root / "scripts" / "decisions_lib.py").write_bytes(DECISIONS_LIB.read_bytes())

    docs = root / "docs"
    docs.mkdir()
    (docs / "decisions.md").write_bytes((REPO_ROOT / "docs" / "decisions.md").read_bytes())

    dst_decisions = docs / "decisions"
    src_decisions = REPO_ROOT / "docs" / "decisions"
    dst_decisions.mkdir()
    for item in src_decisions.iterdir():
        if item.is_dir():
            _copy_tree(item, dst_decisions / item.name)
        else:
            (dst_decisions / item.name).write_bytes(item.read_bytes())


def _copy_tree(src: Path, dst: Path) -> None:
    dst.mkdir(parents=True, exist_ok=True)
    for item in src.rglob("*"):
        rel = item.relative_to(src)
        target = dst / rel
        if item.is_dir():
            target.mkdir(parents=True, exist_ok=True)
        else:
            target.parent.mkdir(parents=True, exist_ok=True)
            target.write_bytes(item.read_bytes())


WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "pr-checks.yml"


def _active_runs(workflow: Path | None = None) -> list[str]:
    """The `run` script of every step the `decisions-guard` job would ACTUALLY execute.

    Parsed with `yaml.safe_load`, and returned WHOLE — not split into lines. Both choices are
    scar tissue.

    Text-scanning for `run:` was round one, and it broke three ways: a `run: |` block
    scalar was invisible; a job or step switched off still read as wired; and `run:` inside
    block-scalar *text* was extracted and executed. Round two parsed the YAML and matched a LINE
    beginning with `PYTHONPATH=.` — and that broke too, with a heredoc:

        run: |
          cat <<'EOF' > /dev/null
          PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check
          EOF

    The matched line is heredoc DATA. The extractor reported the guard as running, and the proof
    executed a command CI does not. Deciding which lines of a shell script are executed requires
    parsing shell, and `fixing-a-parser-bug-introduces-the-next-one` is explicit that this repo has
    lost that argument repeatedly — a regex over shell is not a parser, and round four would find
    round five.

    So the line-level heuristic is WITHDRAWN. The whole `run` script is handed to `bash`, exactly as
    the runner does. The heredoc above then runs, writes to `/dev/null`, checks nothing, and exits
    0 — so the stale-catalog case fails to redden and the proof reports the defect instead of
    stepping around it. No shell parsing, and the ambiguous cases resolve by execution.
    """
    doc = yaml.safe_load((workflow or WORKFLOW).read_text())

    # The workflow's TRIGGERS, before its jobs. A `decisions-guard` job that is perfectly healthy
    # gates nothing if the workflow stopped running on pull requests, and starting at `jobs:` cannot
    # see that. Note `on` parses to the boolean True in YAML 1.1 (the Norway problem's cousin), so
    # the key is looked up both ways rather than assumed.
    triggers = (doc or {}).get("on", (doc or {}).get(True)) or {}
    names = set(triggers) if isinstance(triggers, dict) else {triggers} if isinstance(triggers, str) else set(triggers)
    assert "pull_request" in names, (
        "pr-checks.yml no longer runs on `pull_request`, so NO gate in it — including the catalog "
        f"guard — fires on a PR. Triggers found: {sorted(str(n) for n in names)}"
    )

    jobs = (doc or {}).get("jobs") or {}
    job = jobs.get("decisions-guard")
    assert job is not None, (
        "no `decisions-guard` job in pr-checks.yml. Either it was renamed or it was removed — the "
        f"second is the far more serious finding. Jobs present: {sorted(jobs)}"
    )
    assert not _disabled(job), (
        "the `decisions-guard` job is disabled at the job level "
        f"(if: {job.get('if')!r}, continue-on-error: {job.get('continue-on-error')!r}), so nothing "
        "in it runs — including the catalog guard"
    )
    return [str(step["run"]) for step in (job.get("steps") or []) if step.get("run") and not _disabled(step)]


def _unwrap(value: str) -> str:
    """Strip an `${{ ... }}` expression wrapper, if present, and lowercase.

    Written as a regex over the WHOLE value rather than `.strip("${{ }}")`, which strips a character
    SET — it would turn `"false}"` into `"false"` and reads as though it removed a wrapper it never
    checked for.
    """
    inner = value.strip()
    m = re.fullmatch(r"\$\{\{(.*)\}\}", inner, flags=re.DOTALL)
    if m:
        inner = m.group(1)
    return inner.strip().lower()


def _falsey(value) -> bool:
    """A literal false, however this workflow dialect spells it.

    `if: false`, `if: "false"` and `if: ${{ false }}` all mean never. The middle and last are the
    ones a text comparison misses; the last was a live false green — `${{ false }}` is the ordinary
    spelling in Actions-flavoured YAML, and it read as wired.

    An expression that is merely falsy AT RUN TIME (`if: ${{ github.event_name == 'x' }}`) is not
    decidable here and is deliberately not guessed at.
    """
    if value is False:
        return True
    if not isinstance(value, str):
        return False
    return _unwrap(value) == "false"


def _truthy_literal(value) -> bool:
    if value is True:
        return True
    if not isinstance(value, str):
        return False
    return _unwrap(value) == "true"


def _disabled(node: dict) -> bool:
    """A job or step that cannot fail the run: switched off, or allowed to fail.

    `continue-on-error: true` is the subtle one — the step still runs and still reports, but its
    failure does not fail the job, so it is not a gate.
    """
    return _falsey(node.get("if", True)) or _truthy_literal(node.get("continue-on-error", False))


def _ci_check_command(workflow: Path | None = None) -> list[str]:
    """The catalog step's script, DERIVED from the workflow and executed whole.

    `testing.guard-derives-population-from-source`: a hand-copied command is a second copy of the
    workflow that drifts silently, and this test's whole value is that it runs what CI runs.
    """
    runs = _active_runs(workflow)
    matches = [r for r in runs if "build_decisions_catalog.py" in r and "--check" in r]
    assert matches, (
        "the `decisions-guard` job has no ACTIVE step mentioning "
        "`build_decisions_catalog.py --check`. The catalog guard has stopped running in CI — that "
        f"is the finding, not this test's failure. Active step scripts in that job: {runs}"
    )
    assert len(matches) == 1, f"expected exactly one such step, found {matches}"
    script = matches[0]
    assert "${{" not in script, (
        "the catalog step's script interpolates an Actions expression, which cannot be expanded "
        f"outside the runner — this proof would be executing something else: {script!r}"
    )
    # Substitute the interpreter only where `python3` is a bare command word. A plain
    # `str.replace` rewrites EVERY occurrence, including inside a path — `/usr/bin/python3` would
    # become `/usr/bin/<venv>/bin/python3` and fail with ENOENT, a red blaming the catalog guard for
    # something this line did.
    return ["bash", "-c", re.sub(r"(?<![\w/])python3\b", sys.executable, script)]


def _proof_holds(workflow: Path, repo: Path) -> bool:
    """Does the whole guarantee hold — fresh corpus passes AND stale corpus fails?

    The disablement cases below assert on THIS rather than on whether extraction raises, because
    the ways a guard can stop gating do not all surface at the same place. Deletion and disablement
    surface as a failed extraction; a heredoc or an `echo` surfaces only when the script is run and
    reports success over a stale catalog. One predicate covers both.
    """
    try:
        cmd = _ci_check_command(workflow)
    except AssertionError:
        return False

    readme = repo / "docs" / "decisions" / "README.md"
    original = readme.read_text(encoding="utf-8")
    try:
        readme.write_text(original, encoding="utf-8")
        fresh = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
        if fresh.returncode != 0:
            return False
        readme.write_text(original + "\nSTALE INJECTED LINE\n", encoding="utf-8")
        stale = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
        return stale.returncode != 0
    finally:
        readme.write_text(original, encoding="utf-8")


_STEP = (
    "      - name: Active catalog in sync\n"
    "        run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n"
)


@pytest.mark.parametrize(
    "label,mutate",
    [
        (
            "the step is COMMENTED OUT",
            lambda s: s.replace(
                _STEP,
                "      # - name: Active catalog in sync\n"
                "      #   run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
                1,
            ),
        ),
        ("the step is DELETED", lambda s: s.replace(_STEP, "", 1)),
        (
            "the step is switched off with `if: false`",
            lambda s: s.replace(
                _STEP,
                "      - name: Active catalog in sync\n        if: false\n"
                "        run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
                1,
            ),
        ),
        (
            "the step is switched off with `if: ${{ false }}`",
            lambda s: s.replace(
                _STEP,
                "      - name: Active catalog in sync\n        if: ${{ false }}\n"
                "        run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
                1,
            ),
        ),
        (
            "the step is allowed to fail with `continue-on-error: true`",
            lambda s: s.replace(
                _STEP,
                "      - name: Active catalog in sync\n        continue-on-error: true\n"
                "        run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
                1,
            ),
        ),
        (
            "the whole JOB is switched off with `if: false`",
            lambda s: s.replace(
                "  decisions-guard:\n    name: decisions lifecycle\n    runs-on: small\n"
                "    if: github.event_name == 'pull_request'\n",
                "  decisions-guard:\n    name: decisions lifecycle\n    runs-on: small\n    if: false\n",
                1,
            ),
        ),
        (
            "the command survives only as TEXT in another step's `echo`",
            lambda s: s.replace(
                _STEP,
                "      - name: Note\n        run: |\n"
                "          echo we no longer run: PYTHONPATH=. python3 "
                "scripts/build_decisions_catalog.py --check\n",
                1,
            ),
        ),
        (
            "the WORKFLOW no longer runs on pull requests",
            lambda s: s.replace("on:\n  pull_request:", "on:\n  workflow_dispatch:", 1),
        ),
        (
            "the command survives only as HEREDOC DATA",
            lambda s: s.replace(
                _STEP,
                "      - name: Note\n        run: |\n"
                "          cat <<'EOF' > /dev/null\n"
                "          PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n"
                "          EOF\n",
                1,
            ),
        ),
    ],
)
def test_a_guard_that_stopped_RUNNING_is_DETECTED(tmp_path, label, mutate):
    """Eight ways the catalog guard can stop gating, each of which must be caught.

    Commenting out is one of them, and the only one a text scan detects. The last two
    leave the command in the file, parseable and even matchable — as `echo` argument and as heredoc
    data — which is why the proof executes the step's whole script instead of a line lifted out of it.
    """
    raw = WORKFLOW.read_text()
    mutated = mutate(raw)
    assert mutated != raw, f"the mutation for {label!r} matched nothing; RETARGET it"

    alt = tmp_path / "pr-checks.yml"
    alt.write_text(mutated)
    repo = tmp_path / "repo-copy"
    repo.mkdir()
    _seed_decisions_copy(repo)

    assert not _proof_holds(alt, repo), (
        f"the catalog guard still reported as gating when {label}. CI would run nothing and this "
        "file would report full coverage."
    )


@pytest.mark.parametrize(
    "label,replacement",
    [
        (
            "block scalar",
            "      - name: Active catalog in sync\n        run: |\n"
            "          PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
        ),
        (
            "backslash continuation",
            "      - name: Active catalog in sync\n        run: |\n"
            "          PYTHONPATH=. python3 \\\n"
            "            scripts/build_decisions_catalog.py --check\n",
        ),
        (
            "a leading `set -euo pipefail`",
            "      - name: Active catalog in sync\n        run: |\n"
            "          set -euo pipefail\n"
            "          PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n",
        ),
        (
            "wrapped in a shell block",
            "      - name: Active catalog in sync\n        run: |\n"
            "          if true; then\n"
            "            PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check\n"
            "          fi\n",
        ),
    ],
)
def test_the_step_may_be_REFORMATTED_without_being_flagged(tmp_path, label, replacement):
    """The negative controls for the eight above: a legitimate rewrite must NOT be flagged.

    `run: |` is idiomatic in this very job (the sibling "Validate decision lifecycle" step uses it),
    and the text-scanning version reported that form as "the guard has stopped running in CI" — a
    false red whose message asserts a regression that has not happened. A detector that cannot tell
    a reformat from a removal trains its readers to ignore it, which is the whole subject of #806.

    A line-matching version had to be taught each of these shapes one at a time, and the
    continuation case defeated two rounds of it. Executing the script gets all four for free: the
    question "is the guard still gating" is answered by running it, not by recognising how it was
    written.
    """
    raw = WORKFLOW.read_text()
    mutated = raw.replace(_STEP, replacement, 1)
    assert mutated != raw, "retarget this reformatting; the step's text has changed"

    alt = tmp_path / "pr-checks.yml"
    alt.write_text(mutated)
    repo = tmp_path / "repo-copy"
    repo.mkdir()
    _seed_decisions_copy(repo)

    assert _proof_holds(alt, repo), f"a legitimate reformat ({label}) was reported as a removal"


def test_CLI_subprocess_exits_nonzero_on_a_stale_catalog(tmp_path):
    """The real CI invocation, as a subprocess, against a real corpus.

    This is the `__main__` → `SystemExit(main())` wiring proof: `main()` returning 1 is worthless if
    the process still exits 0, and the workflow step reads nothing but the exit code. It is the
    #751/#719 shape the decision record names — a green result behind wiring that is not connected.

    The FRESH half is not optional decoration. Without it this test passes whenever the subprocess
    dies for any reason at all — an import error, a missing file in the copy, a syntax error — none
    of which is the guard detecting anything. `arbitrary-sample-gives-false-negatives` in reverse:
    a non-zero exit is only evidence when the same harness is shown to exit zero on a clean corpus.
    """
    repo = tmp_path / "repo-copy"
    repo.mkdir()
    _seed_decisions_copy(repo)
    cmd = _ci_check_command()

    fresh = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
    assert fresh.returncode == 0, (
        "the copied corpus does not even pass --check when untouched, so a non-zero exit below "
        f"would be the harness failing rather than the guard firing: {fresh.stdout!r} {fresh.stderr!r}"
    )

    readme = repo / "docs" / "decisions" / "README.md"
    readme.write_text(readme.read_text(encoding="utf-8") + "\nSTALE INJECTED LINE\n", encoding="utf-8")

    stale = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True, timeout=60)
    assert stale.returncode != 0, (
        "the CLI wiring did not surface a stale catalog as a non-zero exit — CI would report "
        f"green over a stale README.md: stdout={stale.stdout!r} stderr={stale.stderr!r}"
    )


# ------------------------------------------------------------------------------------------------
# THE MUTATION PROOF — disarm `main()`'s stale-detection clause alone, --check must stop detecting
# ------------------------------------------------------------------------------------------------


def _load_mutated_module(tmp_path: Path, mutated_source: str):
    mutated_path = tmp_path / "mutated_build_decisions_catalog.py"
    mutated_path.write_text(mutated_source, encoding="utf-8")
    spec = importlib.util.spec_from_file_location("mutated_build_decisions_catalog", mutated_path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def test_MUTATION_disarming_the_stale_comparison_stops_detection(tmp_path, monkeypatch):
    """`testing.guard-ships-with-mutation-proof` (#775).

    `main()`'s ONLY stale-detection logic is the clause `want.strip() != have.strip()`. Disarm that
    clause alone — replace it with the constant `False` in an isolated copy of the module, changing
    nothing else — and `--check` must stop reporting staleness. If it still returns 1, the deny is
    coming from somewhere other than the clause the guard is supposed to hang on, and every test
    above proves nothing about it.
    """
    text = CATALOG_SCRIPT.read_text(encoding="utf-8")
    clause = "want.strip() != have.strip()"
    assert clause in text, (
        "the stale-detection clause has moved or been reworded; RETARGET this mutation at its new "
        "location rather than loosening the string match — a mutation that silently stops mutating "
        "is the exact failure this file exists to catch"
    )
    mutated_source = text.replace(clause, "False", 1)
    assert mutated_source != text and clause not in mutated_source, (
        "the replacement did not change the source, so the mutant is the subject"
    )

    fresh = _current_catalog_text()
    output = tmp_path / "README.md"
    output.write_text(fresh.rstrip("\n") + "\nEXTRA STALE LINE\n", encoding="utf-8")

    # POSITIVE CONTROL FIRST. Without this, the mutation assertion below would pass just as well if
    # `main(["--check"])` never detected anything at all on this input — "the mutant is silent"
    # proves nothing unless the real guard is first shown to be loud on the exact same input.
    monkeypatch.setattr(bc, "OUTPUT", output)
    assert bc.main(["--check"]) == 1, (
        "the UNMUTATED guard did not detect the staleness on this input, so a silent mutant below "
        "would prove nothing about the clause"
    )

    mutant = _load_mutated_module(tmp_path, mutated_source)
    mutant.OUTPUT = output
    assert mutant.main(["--check"]) == 0, (
        "disarming `want.strip() != have.strip()` alone did not stop --check from reporting "
        "staleness, so that clause is not what the guard's exit code hangs on"
    )