ersatztv/scripts/tests/test_ci_image_pin_population.py

"""The CI-image pin guard must see a container job that carries NO pin (ersatztv#774).

WHAT THIS IS PROTECTING. `pr-checks.yml`'s `ci-image-pin` job states the invariant in its own error
text — "Every container: job must pin ersatztv-ci:<7-char-sha>" — and then does not check it. What
it checks is:

    mapfile -t pins < <(grep -oE 'ersatztv-ci:[0-9a-f]+' .gitea/workflows/docker-build.yml \
                        | cut -d: -f2 | sort -u)
    [ "${#pins[@]}" -eq 1 ]

`sort -u` collapses to DISTINCT VALUES, so the count is a property of the pins that are PRESENT. A
job that carries a `container:` block with no `ersatztv-ci:` pin — or no `container:` block at all —
contributes nothing to grep's output, so it cannot move that count. Delete the `container:` block
from `test` and four pins remain: still one distinct value, still green, and a REQUIRED context now
runs on the bare runner instead of the toolchain image. That is ersatztv#774's Family A exactly: a
guard that cannot see the member that is MISSING, because its population is the set of matches
rather than the set of jobs.

THE SPLIT WITH THE SHELL GUARD IS DELIBERATE, and is not two copies of one rule (which would be
#773's Family C). Two different assertions over the same subject:

  * `ci-image-pin` (shell, pr-checks.yml) owns the questions that need GIT HISTORY — does the pin
    resolve to a commit, is it exactly 7 chars, is it the last commit to touch `docker/ci`. A
    pytest cannot answer those without a full clone.
  * this file owns the question that needs the PARSED YAML — is the set of jobs declaring a
    `container:` exactly the set of jobs pinning the image. A shell grep structurally cannot answer
    that, which is why it was never asked.

Neither restates the other, and each says so above the code.
"""

from __future__ import annotations

import copy
import re
from pathlib import Path

import pytest
import yaml

from scripts.tests.tracked_files import tracked_paths

REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOWS_DIR = REPO_ROOT / ".gitea" / "workflows"
WORKFLOW = WORKFLOWS_DIR / "docker-build.yml"
# Resolved against the GIT INDEX rather than `Path.glob` (ersatztv#806), and `*.yaml` alongside
# `*.yml`: Gitea accepts both spellings, so a `.yaml` workflow was structurally invisible to the
# scope check below while reading as covered.
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))


def workflow_files() -> list[Path]:
    """THE WORKFLOW POPULATION, from the git index. Named rather than inline so the shared proof in
    `test_guard_populations_derive_from_git.py` can assert it never admits an untracked file."""
    return tracked_paths(*WORKFLOWS)


# The image repository, without the tag. Matched as a whole path rather than by the bare
# `ersatztv-ci` token so a job pointing at a LOOK-ALIKE registry (a personal fork, a typo'd host)
# is a fault rather than a silent pass — the shell guard's `grep -oE 'ersatztv-ci:[0-9a-f]+'` reads
# the tail of any string and would accept `evil.example/ersatztv-ci:32747a0`.
IMAGE_REPO = "192.168.1.95:3000/timothy/ersatztv-ci"
_PIN = re.compile(rf"^{re.escape(IMAGE_REPO)}:([0-9a-f]+)$")

_DOC = yaml.safe_load(WORKFLOW.read_text())

# THE EXECUTION CLASS IS DECLARED BY THE WORKFLOW, one marker per job, and every set below is
# derived from it (ersatztv#789). What stood here before was `TOOLCHAIN_JOBS` /
# `BARE_RUNNER_JOBS` — two hand-written literals enumerating jobs by name.
#
# WHY A LITERAL WAS THERE AT ALL, because the reason is real and the replacement has to answer it.
# The first version of this file compared `container_jobs(doc)` against `pinned_jobs(doc)` and
# nothing else. That is blind to the mutation that matters most: delete a `container:` block and the
# job leaves BOTH sets together, so the comparison stays balanced and reports green — the very
# fail-open this file exists to close, reproduced one level up. A guard whose expected set shrinks
# in step with the thing it guards is not a guard. So the population needs an ANCHOR that does not
# move when the `container:` block does.
#
# `env.CI_EXECUTION_CLASS` is that anchor, and it is a strictly better one than the literal: it is
# still an independent declaration (deleting a `container:` block leaves the marker behind, so the
# job stays in the expected set and the comparison goes red), but it lives IN THE JOB it describes
# instead of in a Python file three directories away — so it is reachable by anyone reading that job,
# and it moves with the job when the job is renamed or removed.
#
# NOT "in the same diff hunk", which an earlier version of this note claimed. MEASURED distance from
# each `container:` block to its marker: test 6 lines, functional-e2e 6, api-docs 7, format 7 —
# and migrations 58, because the whole `services:` block sits between them. A container-only edit to
# `migrations` shows no marker in its hunk at all. Adjacency is a convenience that mostly holds; the
# property the guard actually rests on is INDEPENDENCE — the marker does not move when the
# `container:` block does.
#
# THE COST, stated rather than left for review to find: proximity cuts both ways. The literal sat in
# a file a careless workflow edit would not touch; the marker sits a handful of lines from the
# `container:` block in four of the five jobs, so deleting both together is one plausible slip
# rather than two deliberate acts. That is
# why this file does not stop at the marker. `test_no_job_needs_the_toolchain_without_declaring_it`
# below derives the requirement a THIRD time — from the job's own steps — and it is independent of
# both the marker and the container block, so removing the pair still goes red. It is also the check
# that answers #789's concrete complaint, which neither the literal nor the marker could: move a
# .NET-dependent step into `build` or `scan` and every set-equality here stays balanced while the
# job now needs a toolchain it does not have.
EXECUTION_CLASS_KEY = "CI_EXECUTION_CLASS"
TOOLCHAIN = "toolchain"
BARE_RUNNER = "bare-runner"
# A VOCABULARY, not a population: these are the two legal values of the marker, not a list of jobs.
# `testing.guard-derives-population-from-source` draws exactly that line — the members are derived,
# the value set they are drawn from is a reviewed policy choice.
EXECUTION_CLASSES = frozenset({TOOLCHAIN, BARE_RUNNER})

# Tools present ONLY in the CI toolchain image, used to derive a job's toolchain requirement from
# its own steps. Drawn from docker/ci/Dockerfile: the image adds the .NET SDK, the two dotnet global
# tools, Node/npm and Playwright on top of the ffmpeg base. Deliberately EXCLUDED are `git`, `jq`,
# `python3`, `curl` and `tar`. The Dockerfile installs them, but they do not DISCRIMINATE: the
# bare-runner lanes provide or provision them too — `small` is git-only and adds Python via
# `actions/setup-python` where a job needs it, and `build` runs on stock `ubuntu-latest`. So a match
# on one says nothing about whether the job needs the toolchain image. MEASURED, not assumed: those
# tokens flag 4 of the 8 jobs, so "would flag every job" would have been the wrong reason and the
# wrong number.
TOOLCHAIN_ONLY_TOOLS = (
    "dotnet",
    "dotnet-ef",
    "node",
    "npm",
    "npx",
    "ffmpeg",
    "ffprobe",
    "playwright",
    "reportgenerator",
)
# LONGEST FIRST. Python alternation is first-match, not longest-match, so a bare `dotnet` listed
# ahead of `dotnet-ef` would match the prefix, fail the trailing `(?![\w.-])` on the hyphen, and
# only then backtrack. Sorting removes the dependence on that backtrack rather than relying on it.
_TOOL_ALTERNATION = "|".join(sorted((re.escape(t) for t in TOOLCHAIN_ONLY_TOOLS), key=len, reverse=True))
# Command position: not part of a longer word and not the tail of a path. `(?<![\w./-])` keeps
# `/usr/share/dotnet/dotnet` from matching, and `(?![\w.-])` keeps `nodejs` and `npm-cache` out.
_TOOL_USE = re.compile(rf"(?<![\w./-])({_TOOL_ALTERNATION})(?![\w.-])")

# Full-line shell comments, stripped before matching. WHAT THIS IS AND IS NOT, because the obvious
# reading overstates it: it is PREVENTIVE, not a fix for a live red. Measured on the current tree,
# NO toolchain token appears anywhere in the `run:` bodies of the three bare-runner jobs, comment
# lines included, so the stripping does not change today's verdict for any job. What it removes is
# the cheapest way to acquire a spurious red later — a bare-runner job whose comment merely MENTIONS
# dotnet — and "a correct guard going red on a correct tree is the fastest route to that guard being
# deleted" is this repo's own finding. The substitution is safe in the other direction too: `.sub`
# on a `^…$` match leaves the newline, so no two lines are ever joined and no NEW match can be
# manufactured.
#
# ONLY full-line comments, and the FULL residue is enumerated rather than papered over, because a
# predicate described as tight stops being re-examined. It is wrong in both directions.
#
#   FALSE POSITIVES (spurious red on a correct tree) — a TRAILING `# …` comment, an
#   `echo "run dotnet build"`, and a heredoc BODY, since stripping is line-based and not shell-aware.
#   FALSE NEGATIVES (a real need, unseen) — an absolute path (`/usr/share/dotnet/dotnet build`); a
#   tool reached only through a script; a `#`-prefixed line inside a heredoc that a later command
#   strips and executes, which is the converse of the heredoc false positive; and indirection
#   through a variable — but MEASURED, and only in one of its two forms: `$SDK_CMD build` is
#   invisible when the assignment comes from a step/job `env:` or an earlier step, while
#   `SDK_CMD=dotnet` written in the SAME body is caught, because the assignment line itself carries
#   the bare token.
#
# Parsing those correctly means parsing shell, and `testing.fixing-a-parser-bug-introduces-the-next-one`
# prices a string predicate over source at three or more rounds. It is not attempted. The residue is
# tolerable because of its DIRECTION at each end. A false positive is a LOUD red naming the job and
# the token, with an obvious remedy (reword the line). A false negative costs a detection this check
# never promised — it is a NECESSARY condition, and the two set comparisons above are the primary
# guards. What none of the three can do is see a job that drops its container block, flips its
# marker AND hides its tools at once; that is the disclosed hole, not this list.
_FULL_LINE_COMMENT = re.compile(r"^[ \t]*#.*$", re.M)


def _jobs(doc) -> dict:
    return doc["jobs"]


def container_jobs(doc) -> set[str]:
    """Every job declaring a `container:`. THE AUTHORITATIVE POPULATION.

    Derived from the parsed workflow, which is the only thing that knows the whole of it. A literal
    list here would reintroduce the defect one file over — correct on the day it was written and
    unable to report the day a sixth job appeared.
    """
    return {name for name, job in _jobs(doc).items() if isinstance(job, dict) and "container" in job}


def pinned_jobs(doc) -> dict[str, str]:
    """job -> pinned tag, for every job whose container image is the CI toolchain image."""
    out = {}
    for name, job in _jobs(doc).items():
        if not isinstance(job, dict):
            continue
        image = str((job.get("container") or {}).get("image", ""))
        m = _PIN.match(image)
        if m:
            out[name] = m.group(1)
    return out


def declared_classes(doc) -> dict[str, str | None]:
    """job -> the execution class it DECLARES, or None where the marker is absent.

    `None` is kept rather than dropped, because a job with no marker is the defect
    `class_declaration_faults` reports and a dict that silently omitted it could not.
    """
    out: dict[str, str | None] = {}
    for name, job in _jobs(doc).items():
        if not isinstance(job, dict):
            continue
        env = job.get("env") or {}
        value = env.get(EXECUTION_CLASS_KEY)
        out[name] = str(value) if value is not None else None
    return out


def class_declaration_faults(doc) -> list[str]:
    """MISSING OR UNKNOWN IS A HARD FAILURE, which is the half of #789 that is easy to skip.

    A marker scheme whose absent value defaults to anything at all is a scheme that stops applying
    the moment someone adds a job and forgets — the failure mode is silence, and it arrives on the
    one job nobody reviewed. So an unmarked job is a fault here, and an unrecognised value is a
    fault too: `CI_EXECUTION_CLASS: container` would otherwise read as 'not toolchain' and move the
    job to the bare-runner side of every comparison below without anyone deciding that.
    """
    faults = []
    for name, value in sorted(declared_classes(doc).items()):
        if value is None:
            faults.append(
                f"job '{name}' declares no {EXECUTION_CLASS_KEY}. Every job must declare one under "
                f"its `env:` — {sorted(EXECUTION_CLASSES)} — so that 'runs on the bare runner' is a "
                "recorded decision rather than an omission."
            )
        elif value not in EXECUTION_CLASSES:
            faults.append(
                f"job '{name}' declares {EXECUTION_CLASS_KEY}: {value!r}, which is not one of "
                f"{sorted(EXECUTION_CLASSES)}. An unrecognised value is rejected rather than treated "
                "as bare-runner, because the silent reading would move the job out of the pin check."
            )
    return faults


def toolchain_declared(doc) -> set[str]:
    """The jobs DECLARING that they run in the toolchain image. The expected set, derived."""
    return {n for n, v in declared_classes(doc).items() if v == TOOLCHAIN}


def _step_bodies(job) -> list[str]:
    """Each step's `run:` body with full-line comments removed. See `_FULL_LINE_COMMENT`."""
    return [
        _FULL_LINE_COMMENT.sub("", s["run"])
        for s in (job.get("steps") or [])
        if isinstance(s, dict) and isinstance(s.get("run"), str)
    ]


def jobs_whose_steps_need_the_toolchain(doc) -> dict[str, set[str]]:
    """job -> the toolchain-only tools its OWN step bodies invoke. A THIRD, independent derivation.

    Independent of both the marker and the `container:` block, which is the point: it is what still
    reddens when a careless edit removes that adjacent pair together, and it is the only one of the
    three that can see #789's concrete failure — a .NET-dependent step MOVED into a bare-runner job,
    where every set-equality stays balanced because no set changed.

    THE FILTER HERE IS LEGITIMATE and the distinction matters, because this file's own rule bans
    filtering a population. `testing.guard-derives-population-from-source` draws the line at what the
    claim is over: filtering to select the SUBJECT of a PER-MEMBER property is fine, because the
    excluded members satisfy it vacuously. The claim below is per-job — 'a job that visibly invokes a
    toolchain tool declares the toolchain class' — and a job invoking none satisfies it with nothing
    to check. It is NOT a completeness claim over jobs; the completeness claims are
    `class_declaration_faults` (every job declares) and `pin_population_faults` (both directions).

    ONE-DIRECTIONAL ON PURPOSE, and this is the limit to state rather than let review find. Step
    text is a NECESSARY condition, never a sufficient one: a job whose ONLY toolchain use sat behind
    a script would be invisible here. MEASURED, because the first version of this note asserted the
    blind spot was live and it is not — all five declared toolchain jobs are detected directly,
    `functional-e2e` included, which runs `dotnet restore` and `npm ci` in its own steps before
    handing off to `scripts/e2e-local.sh`. The blind-spot set is EMPTY today. The converse —
    'declared toolchain, so some step must name a tool' — is still not asserted, because a future
    job could be written that way and a red on a correct tree is how a guard gets deleted.

    THE RESIDUAL THIS LEAVES, which is the one that matters and is not the same sentence. The three
    checks fail together under one plausible edit: drop the `container:` block, flip the marker to
    bare-runner, AND move the tool invocation into a script. Then the two set comparisons stay
    balanced (nothing is in either set) and this check sees no token, so nothing reddens. That is a
    real hole and it is stated rather than argued away. What bounds it is the failure MODE, not the
    probability: the job then runs a missing binary and dies with `dotnet: command not found`, which
    is loud and immediate — unlike ersatztv#774's original defect, where a required check WOULD go
    green on the bare runner having silently skipped the toolchain. (Subjunctive on purpose: that
    hole was found and closed by inspection, and no run of it was ever observed.) Trading a silent pass for a noisy
    crash is the point of the change; it is not a claim that the hole is closed.
    """
    out = {}
    for name, job in _jobs(doc).items():
        if not isinstance(job, dict):
            continue
        found = {m for body in _step_bodies(job) for m in _TOOL_USE.findall(body)}
        if found:
            out[name] = found
    return out


def pin_population_faults(doc) -> list[str]:
    """Set equality in BOTH directions, plus tag agreement. Accumulated, never fail-fast.

    Both directions are reported separately because they are different defects. UNPINNED (a
    container job the guard cannot see) is the fail-open this file exists for. PINNED-BUT-NOT-A-
    CONTAINER-JOB cannot arise from `pinned_jobs` as written, but is computed anyway so that a
    future change to either helper cannot quietly make the comparison one-sided.
    """
    declared = container_jobs(doc)
    pinned = pinned_jobs(doc)
    expected = toolchain_declared(doc)

    faults = []
    # AGAINST THE DECLARED CLASS FIRST. This is the direction the two derived sets cannot cover: a
    # job that loses its `container:` block leaves `declared` and `pinned` together, so their
    # equality survives untouched while the job quietly moves to the bare runner. The marker does
    # not move with the block, so this comparison still notices.
    for name in sorted(expected - set(pinned)):
        faults.append(
            f"job '{name}' declares {EXECUTION_CLASS_KEY}: {TOOLCHAIN} but does not pin the image "
            "(its container: block is missing or points elsewhere) — it is running on the bare "
            "runner"
        )
    for name in sorted(set(pinned) - expected):
        faults.append(f"job '{name}' pins the toolchain image but does not declare {EXECUTION_CLASS_KEY}: {TOOLCHAIN}")

    for name in sorted(declared - set(pinned)):
        image = str((_jobs(doc)[name].get("container") or {}).get("image", ""))
        faults.append(
            f"job '{name}' declares a container: but its image is {image!r}, not "
            f"{IMAGE_REPO}:<sha>. ci-image-pin's grep cannot see this job at all, so the pin it "
            "reports as current says nothing about what this job actually runs in."
        )
    for name in sorted(set(pinned) - declared):
        faults.append(f"job '{name}' pins the image without declaring a container: block")

    tags = set(pinned.values())
    if len(tags) > 1:
        faults.append(
            f"jobs pin DIFFERENT tags: {sorted((n, t) for n, t in pinned.items())}. All container "
            "jobs must run the same toolchain image."
        )
    return faults


# ------------------------------------------------------------------------------------------------
# THE LIVE ASSERTION
# ------------------------------------------------------------------------------------------------


def test_every_container_job_pins_the_CI_toolchain_image():
    faults = pin_population_faults(_DOC)
    assert not faults, (
        "docker-build.yml has a container: job the CI-image pin guard cannot see:\n  "
        + "\n  ".join(faults)
        + "\n\n`ci-image-pin` counts DISTINCT pin strings, so a job with no pin contributes nothing "
        "to that count and passes silently while running on the bare runner. See ersatztv#774."
    )


def test_the_declared_class_and_the_workflow_agree_on_which_jobs_use_the_toolchain():
    """BOTH directions against the DECLARED class — the anchor that does not move (ersatztv#789).

    Left-to-right catches a job silently LOSING its container block (the mutation set equality
    between two derived sets cannot see, because both sides shrink together). Right-to-left catches
    a NEW container job that declared itself bare-runner. Neither direction is optional and the
    messages differ, because the two are opposite mistakes.
    """
    declared = container_jobs(_DOC)
    expected = toolchain_declared(_DOC)
    assert expected - declared == set(), (
        f"these jobs declare {EXECUTION_CLASS_KEY}: {TOOLCHAIN} but no longer declare a container: "
        f"block — {sorted(expected - declared)}. They are now running on the bare runner. If that "
        f"is deliberate, change the marker to {BARE_RUNNER} in the workflow and say why in the PR."
    )
    assert declared - expected == set(), (
        f"these jobs declare a container: but their {EXECUTION_CLASS_KEY} is not {TOOLCHAIN} — "
        f"{sorted(declared - expected)}. Fix the marker, or they will run on an image nothing checks."
    )


def test_no_job_needs_the_toolchain_without_declaring_it():
    """THE THIRD DERIVATION, and the only one that can see a step MOVED into a bare-runner job.

    This is #789's concrete complaint: "move a .NET-dependent step into `build` or `scan` while
    leaving that job in `BARE_RUNNER_JOBS`. Every test stays green while the job now needs the
    toolchain image and does not have it." Nothing above can see it — no set changes, so every
    equality stays balanced. Only reading the job's own steps can.

    Necessary-condition only; `jobs_whose_steps_need_the_toolchain` states why the converse is not
    asserted.
    """
    needed = jobs_whose_steps_need_the_toolchain(_DOC)
    expected = toolchain_declared(_DOC)
    faults = [
        f"job '{name}' invokes {sorted(tools)} in its own steps but declares "
        f"{EXECUTION_CLASS_KEY}: {declared_classes(_DOC).get(name)}"
        for name, tools in sorted(needed.items())
        if name not in expected
    ]
    assert not faults, (
        "these jobs run a tool that exists ONLY in the CI toolchain image, on the bare runner:\n  "
        + "\n  ".join(faults)
        + f"\n\nEither declare {EXECUTION_CLASS_KEY}: {TOOLCHAIN} and add the container: block, or "
        "move the step back. Neither bare-runner lane provides the CI image's toolchain: `small` "
        "is git-only (Python is added by actions/setup-python where a job needs it) and `build` "
        "runs on a stock ubuntu-latest."
    )


def test_every_job_declares_a_known_execution_class():
    """ANTI-VACUITY with teeth: no job may be unclassified, and no value may be improvised.

    A floor like `len(container_jobs) >= 3` would be satisfied by a broken parse that happened to
    find four jobs, and would say nothing about a NEW job appearing in a third state nobody
    considered. Requiring a marker on every job means every job is a decision someone recorded — the
    property the old `TOOLCHAIN_JOBS | BARE_RUNNER_JOBS` partition provided, now held without either
    literal.
    """
    faults = class_declaration_faults(_DOC)
    assert not faults, "docker-build.yml has jobs with no usable execution class:\n  " + "\n  ".join(faults)


# ------------------------------------------------------------------------------------------------
# MUTATION PROOFS — disarm the invariant one way at a time, each must be DETECTED (ersatztv#775)
# ------------------------------------------------------------------------------------------------


def _mutants():
    """(id, mutated doc) for each single-job way the invariant can be broken.

    Every container job in turn, not a sample: the interesting drop is whichever job someone
    actually edits, and proving detection on only the first would prove the case least likely to
    happen (ersatztv#773 §3 Family A, applied to this file's own tests).
    """
    for job in sorted(container_jobs(_DOC)):
        dropped = copy.deepcopy(_DOC)
        del dropped["jobs"][job]["container"]
        yield f"{job}-container-removed", dropped

        unpinned = copy.deepcopy(_DOC)
        unpinned["jobs"][job]["container"]["image"] = "mcr.microsoft.com/dotnet/sdk:10.0"
        yield f"{job}-image-swapped", unpinned

        lookalike = copy.deepcopy(_DOC)
        lookalike["jobs"][job]["container"]["image"] = "evil.example/timothy/ersatztv-ci:32747a0"
        yield f"{job}-lookalike-registry", lookalike

        skewed = copy.deepcopy(_DOC)
        skewed["jobs"][job]["container"]["image"] = f"{IMAGE_REPO}:deadbee"
        yield f"{job}-tag-skewed", skewed

        # The marker flipped while the container: block stays. The mutation the LITERAL registry
        # could also catch, kept so replacing it with a declaration demonstrably lost nothing.
        misdeclared = copy.deepcopy(_DOC)
        misdeclared["jobs"][job].setdefault("env", {})[EXECUTION_CLASS_KEY] = BARE_RUNNER
        yield f"{job}-declared-bare-runner", misdeclared


_MUTANTS = list(_mutants())


@pytest.mark.parametrize("doc", [m for _, m in _MUTANTS], ids=[i for i, _ in _MUTANTS])
def test_a_single_job_losing_its_pin_is_DETECTED(doc):
    """The proof this guard can go red. Without it, `pin_population_faults` returning a constant
    empty list would satisfy the live assertion above and prove nothing — which is how #621 and
    #685 both shipped."""
    assert pin_population_faults(doc), (
        "the population check accepted a workflow in which a container job no longer runs the pinned toolchain image"
    )


def test_the_mutation_set_is_not_empty():
    """The positive control for the parametrisation itself.

    If `container_jobs` ever returned an empty set, `_mutants()` would yield nothing, pytest would
    collect zero cases from the decorator above, and the file would report all-green having proved
    nothing. That is the vacuous-by-sampling shape this whole issue is about, and it is reachable
    here through a single broken helper.
    """
    expected = 5 * len(toolchain_declared(_DOC))
    assert len(_MUTANTS) == expected, (
        f"expected 5 mutations per declared toolchain job ({expected}), got {len(_MUTANTS)}. A floor "
        "rather than an equality here would let a `container_jobs()` that degraded to 3 of 5 jobs "
        "pass while silently testing less — the message would still claim 5 per job."
    )
    assert toolchain_declared(_DOC), (
        "no job declares the toolchain class, so every mutation above is vacuous. Either the marker "
        "key changed or the parse broke."
    )


def test_docker_build_is_the_ONLY_workflow_pinning_the_toolchain_image():
    """This file reads ONE workflow, which is itself a scope mirror needing its own check.

    `WORKFLOW` hardcodes `docker-build.yml`, and the implicit claim — that no other workflow uses
    the toolchain image — mirrors a machine-readable source (the tracked `.gitea/workflows/*.y*ml`)
    that nothing consulted. `renovate.yml` already declares a `container:` with a different image,
    so the shape is live. A future workflow adopting `ersatztv-ci:` would acquire no pin-population
    guard, no single-tag check and no partition, silently, while `pin_population_faults`'s own error
    text claims "All container jobs must run the same toolchain image".

    Found by cold review, which correctly noted this file criticises `MARKED_JOBS` for exactly this
    and then shipped the same shape without even the dated comment `MARKED_JOBS` then carried.
    `MARKED_JOBS` has since been derived (#787), and `TOOLCHAIN_JOBS`/`BARE_RUNNER_JOBS` — the
    literals this file once carried — were replaced by the per-job `env.CI_EXECUTION_CLASS` marker
    in #789. This scope check is what still keeps `WORKFLOW` honest: the marker says which class a
    job is in, never which FILE the guard reads.

    The population comes from the GIT INDEX (ersatztv#806). A `Path.glob` here answered a question
    about the machine rather than about the repo: an untracked scratch workflow left in
    `.gitea/workflows/` would be parsed and could redden this test on one checkout while CI, which
    never sees it, stayed green. The pattern set gained `*.yaml` in the same change — Gitea accepts
    both spellings, so a `.yaml` workflow adopting the toolchain image was invisible here while this
    test read as covering every workflow.

    Checked by PARSING each workflow's `container.image`, not by grepping the file. A text search
    reports `ci-image.yml`, which names the image because it BUILDS and PUSHES it — a producer, not
    a consumer. Grepping would have made this test permanently red on a correct tree, which is the
    fastest route to a correct guard being deleted.
    """
    others = []
    for p in workflow_files():
        if p.name == WORKFLOW.name:
            continue
        doc = yaml.safe_load(p.read_text()) or {}
        for name, job in (doc.get("jobs") or {}).items():
            if not isinstance(job, dict):
                continue
            image = str((job.get("container") or {}).get("image", ""))
            # Keyed on the IMAGE REPOSITORY, not on `_PIN`'s literal-tag match. A job written as
            # `image: <repo>:${{ matrix.tag }}` runs on the toolchain image but fails `_PIN`, so
            # keying on the pin would have let a templated tag slip the whole check — found by cold
            # review, which constructed exactly that. The tag being an expression is itself a fault
            # (nothing could then verify WHICH image ran), so this reports the job either way.
            if image.startswith(f"{IMAGE_REPO}:"):
                others.append(f"{p.name}:{name}")
    assert not others, (
        f"{sorted(others)} run container jobs on the CI toolchain image, but this file only checks "
        f"{WORKFLOW.name}, so they have no pin-population guard at all. Extend the check to cover "
        "them rather than leaving the coverage implied."
    )


def test_the_shell_guards_grep_sees_the_same_tags_the_jobs_run():
    """Ties the two halves together, so they cannot drift into disagreeing about the subject.

    `ci-image-pin` reads the file with a grep for `ersatztv-ci:<hex>`. This compares what that grep
    sees against what the parsed jobs actually run.

    DISTINCT VALUES rather than a count, deliberately. The counts legitimately differ: the file's
    header comment at docker-build.yml:32 documents the pin in prose, so the shell guard's grep
    reads SIX strings where the YAML has five pinned jobs. Asserting on the count would either fail
    today or have to hardcode "+1 for the comment", which breaks the moment a second comment
    mentions the pin.

    What actually has to hold for the shell guard's verdict to be sound is that its `sort -u` set
    equals the set of tags the jobs really run. Comparing the distinct sets says exactly that — and
    as a free side effect it makes the header comment SELF-CHECKING: bump the five image lines and
    forget the comment, and the sets diverge here with a message naming both, instead of the shell
    guard reporting "pins MORE THAN ONE ersatztv-ci tag" and pointing at prose.
    """
    text = WORKFLOW.read_text()
    grepped = {m for m in re.findall(r"ersatztv-ci:([0-9a-f]+)", text)}
    parsed = set(pinned_jobs(_DOC).values())
    assert grepped == parsed, (
        f"ci-image-pin's grep sees the distinct tags {sorted(grepped)} but the parsed container "
        f"jobs run {sorted(parsed)}. A tag mentioned in the file but not run by any job (a stale "
        "header comment) makes the shell guard's 'MORE THAN ONE pin' check fire on prose; a tag "
        "run but not greppable means the shell guard is not checking that job at all."
    )


# ------------------------------------------------------------------------------------------------
# MUTATION PROOFS for the two checks the pin comparison cannot make (ersatztv#789)
# ------------------------------------------------------------------------------------------------


def _declaration_mutants():
    """Every job in turn loses or corrupts its marker. Both must be reported, not defaulted."""
    for job in sorted(_jobs(_DOC)):
        removed = copy.deepcopy(_DOC)
        (removed["jobs"][job].get("env") or {}).pop(EXECUTION_CLASS_KEY, None)
        yield f"{job}-marker-removed", removed

        unknown = copy.deepcopy(_DOC)
        unknown["jobs"][job].setdefault("env", {})[EXECUTION_CLASS_KEY] = "container"
        yield f"{job}-marker-unknown-value", unknown


_DECLARATION_MUTANTS = list(_declaration_mutants())


@pytest.mark.parametrize("doc", [m for _, m in _DECLARATION_MUTANTS], ids=[i for i, _ in _DECLARATION_MUTANTS])
def test_a_missing_or_UNKNOWN_execution_class_is_DETECTED(doc):
    """Proves the hard-failure half of #789 is real rather than declared.

    A marker scheme is worth nothing if an absent marker reads as a default: the job nobody
    remembered to mark is exactly the job nobody reviewed.
    """
    assert class_declaration_faults(doc), "a job with no execution class, or an unrecognised one, was accepted"


def test_the_declaration_mutation_set_is_not_empty():
    """NON-EMPTINESS first: `2N == 2N` holds at N=0 and would pass over an empty parametrisation."""
    assert _jobs(_DOC), (
        "the workflow parsed to ZERO jobs, so every declaration mutation is vacuous and the "
        "parametrised proof above collected nothing."
    )
    expected = 2 * len(_jobs(_DOC))
    assert len(_DECLARATION_MUTANTS) == expected, (
        f"expected 2 declaration mutations per job ({expected}), got {len(_DECLARATION_MUTANTS)}"
    )


def _requirement_mutants():
    """A toolchain-only step MOVED into each bare-runner job in turn — #789's concrete failure.

    Appended as a real step rather than by editing the marker, because the whole point is that this
    defect changes NO set: the job keeps its marker, keeps its lack of a container block, and every
    equality above stays balanced.
    """
    bare = sorted(n for n, v in declared_classes(_DOC).items() if v == BARE_RUNNER)
    for job in bare:
        moved = copy.deepcopy(_DOC)
        moved["jobs"][job].setdefault("steps", []).append(
            {"name": "moved here by the mutation harness", "run": "dotnet build ErsatzTV.sln"}
        )
        yield f"{job}-gains-a-dotnet-step", moved


_REQUIREMENT_MUTANTS = list(_requirement_mutants())


@pytest.mark.parametrize("doc", [m for _, m in _REQUIREMENT_MUTANTS], ids=[i for i, _ in _REQUIREMENT_MUTANTS])
def test_a_toolchain_step_MOVED_into_a_bare_runner_job_is_DETECTED(doc):
    needed = jobs_whose_steps_need_the_toolchain(doc)
    expected = toolchain_declared(doc)
    assert [n for n in needed if n not in expected], (
        "a bare-runner job gained a step invoking a tool that exists only in the toolchain image, "
        "and nothing reported it — which is exactly the failure ersatztv#789 filed"
    )


def test_the_requirement_mutation_set_is_not_empty():
    bare = {n for n, v in declared_classes(_DOC).items() if v == BARE_RUNNER}
    assert bare, "no job declares bare-runner, so the requirement mutations are vacuous"
    assert len(_REQUIREMENT_MUTANTS) == len(bare), (
        f"expected one mutation per bare-runner job ({len(bare)}), got {len(_REQUIREMENT_MUTANTS)}"
    )