ersatztv/scripts/tests/test_ci_image_pin_population.py

"""The CI-image pin guard must see a container job that carries NO pin (ersatztv#774).

WHAT THIS IS PROTECTING. `pr-checks.yml`'s `ci-image-pin` job states the invariant in its own error
text — "Every container: job must pin ersatztv-ci:<7-char-sha>" — and then does not check it. What
it checks is:

    mapfile -t pins < <(grep -oE 'ersatztv-ci:[0-9a-f]+' .gitea/workflows/docker-build.yml \
                        | cut -d: -f2 | sort -u)
    [ "${#pins[@]}" -eq 1 ]

`sort -u` collapses to DISTINCT VALUES, so the count is a property of the pins that are PRESENT. A
job that carries a `container:` block with no `ersatztv-ci:` pin — or no `container:` block at all —
contributes nothing to grep's output, so it cannot move that count. Delete the `container:` block
from `test` and four pins remain: still one distinct value, still green, and a REQUIRED context now
runs on the bare runner instead of the toolchain image. That is ersatztv#774's Family A exactly: a
guard that cannot see the member that is MISSING, because its population is the set of matches
rather than the set of jobs.

THE SPLIT WITH THE SHELL GUARD IS DELIBERATE, and is not two copies of one rule (which would be
#773's Family C). Two different assertions over the same subject:

  * `ci-image-pin` (shell, pr-checks.yml) owns the questions that need GIT HISTORY — does the pin
    resolve to a commit, is it exactly 7 chars, is it the last commit to touch `docker/ci`. A
    pytest cannot answer those without a full clone.
  * this file owns the question that needs the PARSED YAML — is the set of jobs declaring a
    `container:` exactly the set of jobs pinning the image. A shell grep structurally cannot answer
    that, which is why it was never asked.

Neither restates the other, and each says so above the code.
"""

from __future__ import annotations

import copy
import re
from pathlib import Path

import pytest
import yaml

from scripts.tests.tracked_files import tracked_paths

REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOWS_DIR = REPO_ROOT / ".gitea" / "workflows"
WORKFLOW = WORKFLOWS_DIR / "docker-build.yml"
# Resolved against the GIT INDEX rather than `Path.glob` (ersatztv#806), and `*.yaml` alongside
# `*.yml`: Gitea accepts both spellings, so a `.yaml` workflow was structurally invisible to the
# scope check below while reading as covered.
WORKFLOWS = (".gitea/workflows", ("*.yml", "*.yaml"))


def workflow_files() -> list[Path]:
    """THE WORKFLOW POPULATION, from the git index. Named rather than inline so the shared proof in
    `test_guard_populations_derive_from_git.py` can assert it never admits an untracked file."""
    return tracked_paths(*WORKFLOWS)


# The image repository, without the tag. Matched as a whole path rather than by the bare
# `ersatztv-ci` token so a job pointing at a LOOK-ALIKE registry (a personal fork, a typo'd host)
# is a fault rather than a silent pass — the shell guard's `grep -oE 'ersatztv-ci:[0-9a-f]+'` reads
# the tail of any string and would accept `evil.example/ersatztv-ci:32747a0`.
IMAGE_REPO = "192.168.1.95:3000/timothy/ersatztv-ci"
_PIN = re.compile(rf"^{re.escape(IMAGE_REPO)}:([0-9a-f]+)$")

_DOC = yaml.safe_load(WORKFLOW.read_text())

# THE HAND-REVIEWED REGISTRY of jobs that must run inside the CI toolchain image, cross-checked in
# BOTH directions against what the workflow actually declares. This is deliberately a literal, and
# the reason is the one case set equality between two DERIVED sets cannot cover.
#
# The first version of this file compared `container_jobs(doc)` against `pinned_jobs(doc)` and
# nothing else. That is blind to the mutation that matters most: delete a `container:` block and the
# job leaves BOTH sets together, so the comparison stays balanced and reports green — which is the
# very fail-open this file was written to close, reproduced one level up. A guard whose expected set
# shrinks in step with the thing it is guarding is not a guard.
#
# So the population needs one anchor that does NOT move when the workflow does, and a reviewed list
# is the only available one: nothing else in the repo records which jobs are supposed to need the
# toolchain. This is the same shape as `web/src/api/pageSizeCallSites.guard.test.ts` — discovery on
# one side, a reviewed registry on the other, compared both ways — and it is the SCOPE half of
# `testing.guard-derives-population-from-source`, not an exception to it. Editing this set is a
# reviewable act; a job silently losing its container block is not.
TOOLCHAIN_JOBS = frozenset({"test", "migrations", "functional-e2e", "api-docs", "format"})

# `scan` and `build` deliberately run on the bare runner: `scan` is `runs-on: small` and needs only
# python, and `build` drives docker/buildx on the host. Listed here so their ABSENCE above reads as
# a decision rather than an oversight.
BARE_RUNNER_JOBS = frozenset({"scan", "build"})


def _jobs(doc) -> dict:
    return doc["jobs"]


def container_jobs(doc) -> set[str]:
    """Every job declaring a `container:`. THE AUTHORITATIVE POPULATION.

    Derived from the parsed workflow, which is the only thing that knows the whole of it. A literal
    list here would reintroduce the defect one file over — correct on the day it was written and
    unable to report the day a sixth job appeared.
    """
    return {name for name, job in _jobs(doc).items() if isinstance(job, dict) and "container" in job}


def pinned_jobs(doc) -> dict[str, str]:
    """job -> pinned tag, for every job whose container image is the CI toolchain image."""
    out = {}
    for name, job in _jobs(doc).items():
        if not isinstance(job, dict):
            continue
        image = str((job.get("container") or {}).get("image", ""))
        m = _PIN.match(image)
        if m:
            out[name] = m.group(1)
    return out


def pin_population_faults(doc) -> list[str]:
    """Set equality in BOTH directions, plus tag agreement. Accumulated, never fail-fast.

    Both directions are reported separately because they are different defects. UNPINNED (a
    container job the guard cannot see) is the fail-open this file exists for. PINNED-BUT-NOT-A-
    CONTAINER-JOB cannot arise from `pinned_jobs` as written, but is computed anyway so that a
    future change to either helper cannot quietly make the comparison one-sided.
    """
    declared = container_jobs(doc)
    pinned = pinned_jobs(doc)

    faults = []
    # AGAINST THE REGISTRY FIRST. This is the direction the two derived sets cannot cover: a job
    # that loses its `container:` block leaves `declared` and `pinned` together, so their equality
    # survives untouched while the job quietly moves to the bare runner.
    for name in sorted(TOOLCHAIN_JOBS - set(pinned)):
        faults.append(
            f"job '{name}' is registered as needing the CI toolchain image but does not pin it "
            "(its container: block is missing or points elsewhere) — it is running on the bare "
            "runner"
        )
    for name in sorted(set(pinned) - TOOLCHAIN_JOBS):
        faults.append(f"job '{name}' pins the toolchain image but is not in TOOLCHAIN_JOBS")

    for name in sorted(declared - set(pinned)):
        image = str((_jobs(doc)[name].get("container") or {}).get("image", ""))
        faults.append(
            f"job '{name}' declares a container: but its image is {image!r}, not "
            f"{IMAGE_REPO}:<sha>. ci-image-pin's grep cannot see this job at all, so the pin it "
            "reports as current says nothing about what this job actually runs in."
        )
    for name in sorted(set(pinned) - declared):
        faults.append(f"job '{name}' pins the image without declaring a container: block")

    tags = set(pinned.values())
    if len(tags) > 1:
        faults.append(
            f"jobs pin DIFFERENT tags: {sorted((n, t) for n, t in pinned.items())}. All container "
            "jobs must run the same toolchain image."
        )
    return faults


# ------------------------------------------------------------------------------------------------
# THE LIVE ASSERTION
# ------------------------------------------------------------------------------------------------


def test_every_container_job_pins_the_CI_toolchain_image():
    faults = pin_population_faults(_DOC)
    assert not faults, (
        "docker-build.yml has a container: job the CI-image pin guard cannot see:\n  "
        + "\n  ".join(faults)
        + "\n\n`ci-image-pin` counts DISTINCT pin strings, so a job with no pin contributes nothing "
        "to that count and passes silently while running on the bare runner. See ersatztv#774."
    )


def test_the_registry_and_the_workflow_agree_on_which_jobs_use_the_toolchain():
    """BOTH directions against the reviewed registry — the anchor that does not move.

    Left-to-right catches a job silently LOSING its container block (the mutation set equality
    between two derived sets cannot see, because both sides shrink together). Right-to-left catches
    a NEW container job nobody registered. Neither direction is optional and the messages differ,
    because the two are opposite mistakes.
    """
    declared = container_jobs(_DOC)
    assert TOOLCHAIN_JOBS - declared == frozenset(), (
        f"these jobs are registered as needing the CI toolchain image but no longer declare a "
        f"container: block — {sorted(TOOLCHAIN_JOBS - declared)}. They are now running on the bare "
        "runner. If that is deliberate, move them to BARE_RUNNER_JOBS in this file and say why in "
        "the PR."
    )
    assert declared - TOOLCHAIN_JOBS == frozenset(), (
        f"these jobs declare a container: but are not in TOOLCHAIN_JOBS — "
        f"{sorted(declared - TOOLCHAIN_JOBS)}. Add them to the registry so the pin guard covers "
        "them, or they will run on an image nothing checks."
    )


def test_the_registry_partitions_every_job_in_the_workflow():
    """No job may be in neither list. ANTI-VACUITY with teeth, and the reason it is here:

    a floor like `len(container_jobs) >= 3` would be satisfied by a broken parse that happened to
    find four jobs, and would say nothing about a NEW job appearing in a third state nobody
    considered. Partitioning the whole `jobs:` map means every job is a decision someone recorded.
    """
    all_jobs = set(_jobs(_DOC))
    unclassified = all_jobs - TOOLCHAIN_JOBS - BARE_RUNNER_JOBS
    assert not unclassified, (
        f"docker-build.yml has job(s) {sorted(unclassified)} that are in neither TOOLCHAIN_JOBS nor "
        "BARE_RUNNER_JOBS. Every job must be one or the other, so that 'runs on the bare runner' is "
        "always a recorded decision rather than an omission."
    )
    assert not (TOOLCHAIN_JOBS & BARE_RUNNER_JOBS), "a job cannot be in both lists"
    assert all_jobs == TOOLCHAIN_JOBS | BARE_RUNNER_JOBS, (
        f"the registry names jobs that do not exist: {sorted((TOOLCHAIN_JOBS | BARE_RUNNER_JOBS) - all_jobs)}"
    )


# ------------------------------------------------------------------------------------------------
# MUTATION PROOFS — disarm the invariant one way at a time, each must be DETECTED (ersatztv#775)
# ------------------------------------------------------------------------------------------------


def _mutants():
    """(id, mutated doc) for each single-job way the invariant can be broken.

    Every container job in turn, not a sample: the interesting drop is whichever job someone
    actually edits, and proving detection on only the first would prove the case least likely to
    happen (ersatztv#773 §3 Family A, applied to this file's own tests).
    """
    for job in sorted(container_jobs(_DOC)):
        dropped = copy.deepcopy(_DOC)
        del dropped["jobs"][job]["container"]
        yield f"{job}-container-removed", dropped

        unpinned = copy.deepcopy(_DOC)
        unpinned["jobs"][job]["container"]["image"] = "mcr.microsoft.com/dotnet/sdk:10.0"
        yield f"{job}-image-swapped", unpinned

        lookalike = copy.deepcopy(_DOC)
        lookalike["jobs"][job]["container"]["image"] = "evil.example/timothy/ersatztv-ci:32747a0"
        yield f"{job}-lookalike-registry", lookalike

        skewed = copy.deepcopy(_DOC)
        skewed["jobs"][job]["container"]["image"] = f"{IMAGE_REPO}:deadbee"
        yield f"{job}-tag-skewed", skewed


_MUTANTS = list(_mutants())


@pytest.mark.parametrize("doc", [m for _, m in _MUTANTS], ids=[i for i, _ in _MUTANTS])
def test_a_single_job_losing_its_pin_is_DETECTED(doc):
    """The proof this guard can go red. Without it, `pin_population_faults` returning a constant
    empty list would satisfy the live assertion above and prove nothing — which is how #621 and
    #685 both shipped."""
    assert pin_population_faults(doc), (
        "the population check accepted a workflow in which a container job no longer runs the pinned toolchain image"
    )


def test_the_mutation_set_is_not_empty():
    """The positive control for the parametrisation itself.

    If `container_jobs` ever returned an empty set, `_mutants()` would yield nothing, pytest would
    collect zero cases from the decorator above, and the file would report all-green having proved
    nothing. That is the vacuous-by-sampling shape this whole issue is about, and it is reachable
    here through a single broken helper.
    """
    expected = 4 * len(TOOLCHAIN_JOBS)
    assert len(_MUTANTS) == expected, (
        f"expected 4 mutations per registered job ({expected}), got {len(_MUTANTS)}. A floor rather "
        "than an equality here would let a `container_jobs()` that degraded to 3 of 5 jobs pass "
        "while silently testing less — the message would still claim 4 per job."
    )


def test_docker_build_is_the_ONLY_workflow_pinning_the_toolchain_image():
    """This file reads ONE workflow, which is itself a scope mirror needing its own check.

    `WORKFLOW` hardcodes `docker-build.yml`, and the implicit claim — that no other workflow uses
    the toolchain image — mirrors a machine-readable source (the tracked `.gitea/workflows/*.y*ml`)
    that nothing consulted. `renovate.yml` already declares a `container:` with a different image,
    so the shape is live. A future workflow adopting `ersatztv-ci:` would acquire no pin-population
    guard, no single-tag check and no partition, silently, while `pin_population_faults`'s own error
    text claims "All container jobs must run the same toolchain image".

    Found by cold review, which correctly noted this file criticises `MARKED_JOBS` for exactly this
    and then shipped the same shape without even the dated comment `MARKED_JOBS` carries.

    The population comes from the GIT INDEX (ersatztv#806). A `Path.glob` here answered a question
    about the machine rather than about the repo: an untracked scratch workflow left in
    `.gitea/workflows/` would be parsed and could redden this test on one checkout while CI, which
    never sees it, stayed green. The pattern set gained `*.yaml` in the same change — Gitea accepts
    both spellings, so a `.yaml` workflow adopting the toolchain image was invisible here while this
    test read as covering every workflow.

    Checked by PARSING each workflow's `container.image`, not by grepping the file. A text search
    reports `ci-image.yml`, which names the image because it BUILDS and PUSHES it — a producer, not
    a consumer. Grepping would have made this test permanently red on a correct tree, which is the
    fastest route to a correct guard being deleted.
    """
    others = []
    for p in workflow_files():
        if p.name == WORKFLOW.name:
            continue
        doc = yaml.safe_load(p.read_text()) or {}
        for name, job in (doc.get("jobs") or {}).items():
            if not isinstance(job, dict):
                continue
            image = str((job.get("container") or {}).get("image", ""))
            # Keyed on the IMAGE REPOSITORY, not on `_PIN`'s literal-tag match. A job written as
            # `image: <repo>:${{ matrix.tag }}` runs on the toolchain image but fails `_PIN`, so
            # keying on the pin would have let a templated tag slip the whole check — found by cold
            # review, which constructed exactly that. The tag being an expression is itself a fault
            # (nothing could then verify WHICH image ran), so this reports the job either way.
            if image.startswith(f"{IMAGE_REPO}:"):
                others.append(f"{p.name}:{name}")
    assert not others, (
        f"{sorted(others)} run container jobs on the CI toolchain image, but this file only checks "
        f"{WORKFLOW.name}, so they have no pin-population guard at all. Extend the check to cover "
        "them rather than leaving the coverage implied."
    )


def test_the_shell_guards_grep_sees_the_same_tags_the_jobs_run():
    """Ties the two halves together, so they cannot drift into disagreeing about the subject.

    `ci-image-pin` reads the file with a grep for `ersatztv-ci:<hex>`. This compares what that grep
    sees against what the parsed jobs actually run.

    DISTINCT VALUES rather than a count, deliberately. The counts legitimately differ: the file's
    header comment at docker-build.yml:32 documents the pin in prose, so the shell guard's grep
    reads SIX strings where the YAML has five pinned jobs. Asserting on the count would either fail
    today or have to hardcode "+1 for the comment", which breaks the moment a second comment
    mentions the pin.

    What actually has to hold for the shell guard's verdict to be sound is that its `sort -u` set
    equals the set of tags the jobs really run. Comparing the distinct sets says exactly that — and
    as a free side effect it makes the header comment SELF-CHECKING: bump the five image lines and
    forget the comment, and the sets diverge here with a message naming both, instead of the shell
    guard reporting "pins MORE THAN ONE ersatztv-ci tag" and pointing at prose.
    """
    text = WORKFLOW.read_text()
    grepped = {m for m in re.findall(r"ersatztv-ci:([0-9a-f]+)", text)}
    parsed = set(pinned_jobs(_DOC).values())
    assert grepped == parsed, (
        f"ci-image-pin's grep sees the distinct tags {sorted(grepped)} but the parsed container "
        f"jobs run {sorted(parsed)}. A tag mentioned in the file but not run by any job (a stale "
        "header comment) makes the shell guard's 'MORE THAN ONE pin' check fire on prose; a tag "
        "run but not greppable means the shell guard is not checking that job at all."
    )