timothy/ersatztv
1
0
Fork 0
Code Issues 51 Pull Requests 5 Actions Packages Projects Releases 9 Wiki Activity

fix(887): the image build builds the SPA and does not test it

Browse Source 
`docker/Dockerfile`'s web-build stage is gitless twice over — the build context is
`web/` + `design-system/` so there is no `.git`, and `node:22-bookworm-slim` ships no
git binary. Members of the SPA suite need one or the other, so running the suite there
required naming the ones that cannot run. That list was a population nothing derived:
#883 added a third member without updating the hand-written pair of `--exclude`s, and
because `Build & push image (amd64)` is `if: github.event_name != 'pull_request'` the
resulting red was unreachable on a PR. It landed on `main` and on the `v*` tag path
instead — every image build failed, `:latest` stopped being republished, and a release
cut would have failed at the image build.

Adding a third `--exclude` re-arms the trap, so the list is removed rather than
extended: the stage now lints, typechecks and BUILDS the SPA, and the suite runs once,
unfiltered, in `docker-build.yml`'s `test` job on a real checkout. `build` carries
`needs: [test, migrations, scan]`, so no image is published past a red suite.

`scripts/tests/test_image_build_delegates_the_spa_suite.py` holds both halves — the
negative one alone would be satisfied by deleting the `needs:` edge. Three populations,
all derived: tracked Dockerfiles and workflows from the git index, and which npm scripts
ARE the suite from `web/package.json` (so `test` is in and the Playwright `test:ui-e2e`
is out, with no exemption list). Publishing jobs come from the `docker/build-push-action`
step and the Dockerfile each builds from that step's own `file:` input, which is why
`ci-image.yml` is out of scope by derivation rather than by an entry that would outlive
its reason.

Four mutants witnessed red, each by the intended test: a filtered suite run put back
into the Dockerfile, the `needs:` edge deleted, and the gating run narrowed in both the
block and the single-line `run:` step forms.

Refs: #887
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019T79beF1Ufid3dXju4yqkF
This commit is contained in:
timothy
2026-08-30 13:37:58 +02:00
co-authored by Claude Opus 5
parent cf5f42edf9
commit febaad77d7
9 changed files with 858 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/tests/mutation_manifest.py
+18
View File
@@ -403,6 +403,24 @@ MUTATIONS: tuple[Mutation, ...] = (
"that it moved). Disarming the comparison inside the script instead would prove the second "
"and say nothing about the first, which is the half that was missing.",
),
Mutation(
guard="scripts/tests/test_image_build_delegates_the_spa_suite.py",
target="docker/Dockerfile",
clause="RUN npm run lint && npm run typecheck && npm run build",
replacement="RUN npm run lint && npm run typecheck && npm test -- --run && npm run build",
proof="test_image_build_delegates_the_spa_suite.py::test_no_DOCKERFILE_runs_the_spa_suite",
granularity=CLAUSE,
expect="runs the SPA vitest suite, but its build context has no git",
why="The target is not the guard, per the checker-guard exception: the guard IS a test, so "
"disarming it makes it absent rather than red, and the mutation goes into the guarded "
"ARTIFACT. The clause is the web-build stage's whole command line, and the replacement is "
"the defect itself rather than a caricature of it — ersatztv#883 put a vitest run back into "
"a gitless stage, and every image build failed from that commit until #887. Mutating the "
"detector's own token list instead would prove only that the detector matches its own "
"constants; this proves it reads the real Dockerfile and reports the real shape. Note the "
"mutant is UNFILTERED: a filtered run is what actually broke, but the guard forbids the "
"class, and mutating to the narrower case would leave the broader one unproven.",
),
)
scripts/tests/test_guard_populations_derive_from_git.py
+3
View File
@@ -43,6 +43,7 @@ from scripts.tests import test_ci_image_pin_population as image_pin
from scripts.tests import test_ci_status_context_uniqueness as ctx_unique
from scripts.tests import test_guard_inventory as guard_inventory
from scripts.tests import test_hook_fire_log as hook_fire
from scripts.tests import test_image_build_delegates_the_spa_suite as image_build
from scripts.tests import test_pr_changed_files as pr_changed
from scripts.tests import test_remote_state_inventory as remote_state
from scripts.tests import test_workflow_job_guards as job_guards
@@ -108,6 +109,8 @@ def _as_relative_strings(members) -> set[str]:
DERIVATIONS = (
("test_guard_inventory.derived_guard_files", guard_inventory.derived_guard_files, 25),
("test_hook_fire_log.hook_scripts", hook_fire.hook_scripts, 10),
("test_image_build_delegates_the_spa_suite.dockerfiles", image_build.dockerfiles, 4),
("test_image_build_delegates_the_spa_suite.workflow_files", image_build.workflow_files, 5),
("test_ci_image_pin_population.workflow_files", image_pin.workflow_files, 5),
("test_ci_status_context_uniqueness.workflow_files", ctx_unique.workflow_files, 5),
("test_remote_state_inventory.derived_population", remote_state.derived_population, 40),
scripts/tests/test_image_build_delegates_the_spa_suite.py
+738
View File
@@ -0,0 +1,738 @@
"""The SPA vitest suite never runs inside an image build, and the image is gated on a job that does
run it, unfiltered (ersatztv#887).
WHAT THIS IS PROTECTING. `docker/Dockerfile`'s `web-build` stage is gitless twice over: its build
context is `web/` + `design-system/`, so there is no `.git`, and `node:22-bookworm-slim` ships no
git binary. Members of the SPA suite need one or the other — the `web/src/api/*.guard.test.ts`
guards derive their file population from `git ls-files` and refuse to fall back to a directory walk
(`testing.guard-derives-population-from-source`), and `trackedSourceFiles.realgit.test.ts` builds a
real temporary repository. Running the suite there anyway therefore requires a list of the members
that cannot run, and that list is a population nothing derives: ersatztv#883 added a third member
without updating a hand-written pair of `--exclude`s, and every image build failed from that commit
on. The red was structurally unreachable on a PR — `Build & push image (amd64)` carries
`if: github.event_name != 'pull_request'` — so it landed on `main` and on the `v*` tag path, where a
release cut fails at the image build and `:latest` stops being republished.
The fix removed the list rather than extending it. This guard is what keeps it removed, and what
holds up the safety argument that licensed removing it, because that argument is a claim about two
OTHER files and would otherwise rot silently in a comment.
THE INVARIANT, in the two halves it actually has:
* NEGATIVE — no Dockerfile runs the suite. Not "no `--exclude` appears": an unfiltered suite run
in a gitless stage fails just as hard, and a filtered one is the trap itself. The whole class is
excluded rather than a spelling of it.
* POSITIVE — every job that publishes an image built from a Dockerfile CARRYING THE SPA SOURCE is
transitively gated on a job that runs the suite, and that run is unfiltered. Without this half
the negative half is a hole: deleting the suite run from the Dockerfile and deleting
`needs: [test, ...]` from `build` would both be green.
POPULATIONS, all derived — never a filesystem walk and never a hand-written list
(`testing.guard-derives-population-from-source`, ersatztv#806):
* Dockerfiles and workflows: the git index, via `tracked_files`. Both the `*.yml` and `*.yaml`
workflow spellings, for the reason `test_workflow_persist_credentials.py` states — Gitea accepts
either, so a `.yaml` workflow would otherwise be structurally invisible.
* WHICH npm SCRIPTS ARE THE SUITE: read out of `web/package.json`, by finding the scripts whose
body invokes the vitest binary. `npm test` is a suite run because `"test": "vitest"` is; the
sibling `"test:ui-e2e": "playwright test"` is not, and no exemption says so. A guess in either
direction here is a live defect — treating `test:ui-e2e` as the suite would demand a gate on a
Playwright run, and hardcoding the name `test` would go quiet the day the script is renamed.
* Image-publishing jobs: every job with a `docker/build-push-action` step, and the Dockerfile it
builds is read out of that step's `file:` input rather than hardcoded here. That is what keeps
a renamed Dockerfile red instead of silently uncovered, and it is why `ci-image.yml` — which
publishes the CI toolchain image from `docker/ci/Dockerfile`, a Dockerfile that carries no SPA
source — is correctly out of scope WITHOUT an exemption entry. An exemption outlives its reason;
a derivation does not.
WHY A SHELL-POSITION PARSER RATHER THAN A REGEX. The first version matched `vitest` anywhere in a
command and reported `echo the vitest suite does not run here` as an invocation. A predicate that
cannot tell a command from prose about a command produces false reds in exactly the files that
explain the rule, and those get silenced. Commands are split on shell separators and only the HEAD
of each segment is consulted, so a runner named in an argument or in prose is not a run.
WHAT IS NOT COVERED, stated rather than implied:
* A line whose FIRST non-space character is `#` is treated as inert, in both file kinds. That is
right for a Dockerfile comment and for a shell comment at the start of a line inside a `run:`
body, and it is what lets `docker/Dockerfile`'s own explanatory comment name the thing it
forbids. A suite invocation appended after a `#` on a line that also carries code is NOT a
comment and is still seen, because only a leading `#` is stripped.
* In a Dockerfile only `RUN` executes at build time, so only `RUN` is read. A `CMD`/`ENTRYPOINT`
that runs the suite would run in the shipped container, which is a different (and absent)
problem; it is out of scope rather than overlooked.
* A `RUN` that invokes a SCRIPT which in turn runs the suite is invisible to this — the parser
reads commands, not what they call. No such script exists today; the failure mode is a missed
detection, not a false green about the gate, because the positive half is derived independently.
* An image published by something other than `docker/build-push-action` — a raw `docker push` in a
`run:` body — is outside the publishing-job derivation. `test_no_run_BODY_builds_or_pushes_an_image`
asserts that shape does not exist rather than leaving it uncovered.
* A genuinely new value-taking vitest flag is reported as a spec filter, because every non-flag
token that is not the value of a KNOWN value-taking flag is treated as one. That is a false red
a reader fixes by naming the flag here — the safe direction.
"""
from __future__ import annotations
import json
import re
import shlex
from pathlib import Path
import yaml
from scripts.tests import tracked_files
from scripts.tests.tracked_files import tracked_paths
REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOW_DIR = ".gitea/workflows"
WORKFLOWS = (WORKFLOW_DIR, ("*.yml", "*.yaml"))
PACKAGE_JSON = "web/package.json"
# The action every image publish in this repo goes through. Matched on the last two path segments
# for the reason `test_workflow_persist_credentials.py` gives: Gitea accepts a full action URL, and
# a host is not required to contain a dot, so no hostname heuristic is safe.
PUBLISH_ACTION = "docker/build-push-action"
# Shell tokens that end one command and start another. `shlex.split` returns each of these as its
# own token, which is what makes segmenting possible without a shell grammar.
SEPARATORS = frozenset({"&&", "||", ";", "|", "&"})
# Prefixes that wrap a command without being one. Skipped so `env FOO=1 npm test` is still a run.
WRAPPERS = frozenset({"env", "sudo", "time", "exec", "command", "nice", "ionice"})
# Dockerfile instructions. Only `RUN` executes during a build; the rest are read as not-a-command so
# a `COPY` whose argument happens to be a runner name is not mistaken for one.
DOCKERFILE_INSTRUCTIONS = frozenset(
{
"ADD",
"ARG",
"CMD",
"COPY",
"ENTRYPOINT",
"ENV",
"EXPOSE",
"FROM",
"HEALTHCHECK",
"LABEL",
"MAINTAINER",
"ONBUILD",
"RUN",
"SHELL",
"STOPSIGNAL",
"USER",
"VOLUME",
"WORKDIR",
}
)
# Flags that narrow WHICH specs run. Every one of these reintroduces the defect: the set of specs
# that ran becomes a thing someone maintains by hand, and nothing compares it to the set that exists.
NARROWING_FLAGS = frozenset(
{"--exclude", "--include", "--dir", "--project", "--shard", "--changed", "--related", "--testNamePattern", "-t"}
)
# Flags whose NEXT token is a value rather than a spec filter, so `--reporter verbose` is not read as
# a filter. Deliberately short — see the docstring's last residual.
VALUE_TAKING_FLAGS = frozenset(
{
"--reporter",
"--outputFile",
"--config",
"--root",
"--environment",
"--pool",
"--maxWorkers",
"--minWorkers",
"--testTimeout",
"--hookTimeout",
"--retry",
"--coverage.reporter",
"--coverage.provider",
}
)
_ENV_ASSIGNMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
def _canonical_action(uses: str) -> str:
"""The last two path segments of a `uses:` value, without its `@version`.
Same normalisation as `test_workflow_persist_credentials._canonical_action`, and deliberately a
second copy rather than an import: that guard's population is checkout steps, this one's is
publish steps, and sharing the helper would mean a narrowing there silently narrows here too.
"""
ref = uses.strip().strip("'\"").rsplit("@", 1)[0].strip().lower()
ref = re.sub(r"^https?://", "", ref)
segments = [seg for seg in ref.split("/") if seg]
if segments:
segments[-1] = re.sub(r"\.git$", "", segments[-1])
return "/".join(segments[-2:]) if len(segments) >= 2 else ref
def logical_commands(text: str) -> list[tuple[int, str]]:
"""`(1-based line number, command)` per command line: comments dropped, `\\` continuations joined.
Both normalisations are load-bearing. Dropping leading-`#` lines is what lets a Dockerfile
explain the rule it enforces without tripping it. Joining backslash continuations is what makes
a multi-line invocation ONE command with its flags attached — the shape the removed `--exclude`
pair had, and the shape a re-added one would have. Scanning raw lines instead would read
`npm test -- --run` as unfiltered and report it clean.
"""
commands: list[tuple[int, str]] = []
buffer = ""
start = 0
for number, raw in enumerate(text.splitlines(), start=1):
if raw.lstrip().startswith("#"):
continue
if not buffer:
start = number
stripped = raw.rstrip()
if stripped.endswith("\\"):
buffer += stripped[:-1] + " "
continue
buffer += stripped
if buffer.strip():
commands.append((start, buffer.strip()))
buffer = ""
if buffer.strip():
commands.append((start, buffer.strip()))
return commands
def command_segments(command: str) -> list[list[str]]:
"""The shell segments of one command, as token lists, or `[]` if nothing executes.
A leading Dockerfile instruction is consumed: `RUN` yields its command, every other instruction
yields nothing, since only `RUN` runs during a build.
"""
try:
tokens = shlex.split(command)
except ValueError:
# An unbalanced quote is not a command anyone can reason about. Returning the raw split
# keeps it VISIBLE to the detectors rather than silently dropping it.
tokens = command.split()
if not tokens:
return []
# A YAML step written on one line — `- run: npm test -- --run` — is a command, and roughly half
# the `run:` steps in this repo are written that way. Without this the raw-text cross-check below
# reads `run:` as the program and finds nothing, so a single-line invocation would be invisible
# to every text-based assertion while the block form was caught.
if tokens[0] == "-":
tokens = tokens[1:]
if tokens and tokens[0] == "run:":
tokens = tokens[1:]
if not tokens:
return []
head = tokens[0]
if head.isupper() and head in DOCKERFILE_INSTRUCTIONS:
if head != "RUN":
return []
tokens = tokens[1:]
segments: list[list[str]] = []
current: list[str] = []
for token in tokens:
if token in SEPARATORS:
if current:
segments.append(current)
current = []
continue
current.append(token)
if current:
segments.append(current)
return segments
def _strip_prefixes(tokens: list[str]) -> list[str]:
index = 0
while index < len(tokens) and (_ENV_ASSIGNMENT.match(tokens[index]) or tokens[index] in WRAPPERS):
index += 1
return tokens[index:]
def direct_vitest_args(tokens: list[str]) -> list[str] | None:
"""The arguments of a segment that invokes the vitest BINARY, or `None`.
Covers `vitest …` (npm puts `node_modules/.bin` on PATH for a script body) and
`npx/npm exec/pnpm exec/yarn dlx vitest …`. Kept separate from the npm-script route below so the
`web/package.json` scan can use it without recursing through the script table it is building.
"""
tokens = _strip_prefixes(tokens)
if not tokens:
return None
if tokens[0] == "vitest":
return tokens[1:]
if tokens[0] in {"npx", "pnpm", "npm", "yarn"}:
index = 1
if tokens[0] != "npx":
if index >= len(tokens) or tokens[index] not in {"exec", "dlx"}:
return None
index += 1
while index < len(tokens) and tokens[index] in {"--", "-y", "--yes", "--no-install"}:
index += 1
if index < len(tokens) and tokens[index] == "vitest":
return tokens[index + 1 :]
return None
def vitest_scripts() -> set[str]:
"""The `web/package.json` script names whose body runs the vitest binary.
This is the derivation that decides what `npm test` and `npm run X` mean. `"test": "vitest"` is
in; `"test:ui-e2e": "playwright test"` is not, and needs no exemption to be left out.
"""
scripts = json.loads((REPO_ROOT / PACKAGE_JSON).read_text(encoding="utf-8")).get("scripts", {})
named: set[str] = set()
for name, body in scripts.items():
if not isinstance(body, str):
continue
if any(direct_vitest_args(segment) is not None for segment in command_segments(body)):
named.add(str(name))
return named
def suite_args(tokens: list[str], scripts: set[str]) -> list[str] | None:
"""The arguments of a segment that runs the SPA suite, or `None` if it does not run it."""
direct = direct_vitest_args(tokens)
if direct is not None:
return direct
tokens = _strip_prefixes(tokens)
if not tokens or tokens[0] not in {"npm", "pnpm", "yarn", "bun"}:
return None
index = 1
if index < len(tokens) and tokens[index] in {"run", "run-script"}:
index += 1
if index >= len(tokens) or tokens[index] not in scripts:
return None
return tokens[index + 1 :]
def suite_invocations(text: str, scripts: set[str] | None = None) -> list[tuple[int, str]]:
"""Every command in `text` that runs the SPA vitest suite."""
scripts = vitest_scripts() if scripts is None else scripts
found: list[tuple[int, str]] = []
for number, command in logical_commands(text):
if any(suite_args(segment, scripts) is not None for segment in command_segments(command)):
found.append((number, command))
return found
def narrowing_arguments(command: str, scripts: set[str] | None = None) -> list[str]:
"""The arguments in `command` that restrict WHICH specs run, or `[]` when it runs all of them."""
scripts = vitest_scripts() if scripts is None else scripts
found: list[str] = []
for segment in command_segments(command):
args = suite_args(segment, scripts)
if args is None:
continue
index = 0
while index < len(args):
argument = args[index]
index += 1
if argument == "--":
continue
base = argument.split("=", 1)[0]
if base in NARROWING_FLAGS:
found.append(argument)
if "=" not in argument and index < len(args) and not args[index].startswith("-"):
found.append(args[index])
index += 1
continue
if base in VALUE_TAKING_FLAGS and "=" not in argument:
index += 1 # its value is not a filter
continue
if argument.startswith("-"):
continue
found.append(argument)
return found
def dockerfiles() -> list[str]:
"""Tracked Dockerfiles, by repo-relative path, in every spelling docker recognises.
`tracked_files._git_ls_files()` through the MODULE, never a `from … import` binding. The shared
proof in `test_guard_populations_derive_from_git.py` narrows the index by patching that module
attribute, and a name bound at import time is not reachable from there — the derivation would go
on returning every Dockerfile after git stopped tracking them, and the proof would say so. A
binding captured before the patch is the same shape that file warns about one block down.
"""
return sorted(
path
for path in tracked_files._git_ls_files()
if path.rpartition("/")[2].startswith("Dockerfile") or path.endswith(".Dockerfile")
)
def workflow_files() -> list[Path]:
return tracked_paths(*WORKFLOWS)
def _read(rel: str) -> str:
return (REPO_ROOT / rel).read_text(encoding="utf-8")
def carries_spa_source(rel: str) -> bool:
"""Does this Dockerfile copy the SPA source tree into the build?
This is what makes a suite run there possible at all, and it is what separates
`docker/Dockerfile` from `docker/ci/Dockerfile` — which installs npm packages globally but never
copies `web/`. Derived from the `COPY` instructions rather than from a list of Dockerfile paths.
Two shapes count. `COPY web/… ` names the tree; `COPY . …` carries it without naming it, and the
question is what the stage HAS, not what it says. No Dockerfile here uses the second form today —
it is matched anyway because the gate assertion SKIPS every Dockerfile this answers False for, so
a miss here drops an image out of the check silently rather than reddening.
A `COPY --from=<stage>` is not a context copy: its source is another stage, so `--from=build .`
would not bring the repo in. Flags are dropped before the source argument is read.
"""
for _, command in logical_commands(_read(rel)):
tokens = command.split()
if not tokens or tokens[0].upper() != "COPY":
continue
arguments = [token for token in tokens[1:] if not token.startswith("--")]
if len(arguments) < 2:
continue
if any(token.startswith("--from=") for token in tokens[1:]):
continue
sources = arguments[:-1]
if any(source in {".", "./"} or re.match(r"^\.?/?web/", source) for source in sources):
return True
return False
def _jobs(doc: object) -> dict[str, dict]:
jobs = doc.get("jobs") if isinstance(doc, dict) else None
if not isinstance(jobs, dict):
return {}
return {str(key): value for key, value in jobs.items() if isinstance(value, dict)}
def _steps(job: dict) -> list[dict]:
steps = job.get("steps")
return [step for step in steps if isinstance(step, dict)] if isinstance(steps, list) else []
def _run_bodies(job: dict) -> list[str]:
return [step["run"] for step in _steps(job) if isinstance(step.get("run"), str)]
def _needs(job: dict) -> list[str]:
needs = job.get("needs")
if isinstance(needs, str):
return [needs]
return [str(item) for item in needs] if isinstance(needs, list) else []
def _transitive_needs(job_id: str, jobs: dict[str, dict]) -> set[str]:
seen: set[str] = set()
frontier = list(_needs(jobs.get(job_id, {})))
while frontier:
current = frontier.pop()
if current in seen:
continue
seen.add(current)
frontier.extend(_needs(jobs.get(current, {})))
return seen
def publishing_jobs() -> list[tuple[str, str, str, dict[str, dict]]]:
"""`(workflow rel path, job id, Dockerfile rel path, that workflow's jobs)` per publishing job.
The Dockerfile comes from the step's own `file:` input, defaulting to docker's own default
(`Dockerfile` at the context root) when the input is absent, so a step that stops naming one is
still resolved rather than dropped from the population.
"""
found: list[tuple[str, str, str, dict[str, dict]]] = []
for path in workflow_files():
rel = path.relative_to(REPO_ROOT).as_posix()
jobs = _jobs(yaml.safe_load(path.read_text(encoding="utf-8")))
for job_id, job in jobs.items():
for step in _steps(job):
uses = step.get("uses")
if not isinstance(uses, str) or _canonical_action(uses) != PUBLISH_ACTION:
continue
with_block = step.get("with")
named = with_block.get("file") if isinstance(with_block, dict) else None
dockerfile = str(named or "Dockerfile").strip()
found.append((rel, job_id, re.sub(r"^\./", "", dockerfile), jobs))
return found
# ------------------------------------------------------------------------------------------------
# The invariant
# ------------------------------------------------------------------------------------------------
def test_no_DOCKERFILE_runs_the_spa_suite() -> None:
"""The negative half. A gitless stage runs no vitest suite, filtered or not."""
scripts = vitest_scripts()
faults: list[str] = []
for rel in dockerfiles():
for number, command in suite_invocations(_read(rel), scripts):
faults.append(
f"{rel}:{number} runs the SPA vitest suite, but its build context has no git and "
f"`node:22-bookworm-slim` ships no git binary, so the suite's git-dependent members "
f"cannot run there: `{command.strip()}`. Excluding them by name is the defect this "
f"guard exists to prevent — that list is a population nothing derives, it went "
f"stale in ersatztv#883, and the resulting red is unreachable on a PR, so it lands "
f"on `main` and on the release tag. The suite runs in `docker-build.yml`'s `test` "
f"job, which gates the image; delete this invocation rather than filtering it "
f"(ersatztv#887)."
)
assert not faults, "\n".join(faults)
def test_every_IMAGE_PUBLISHING_job_is_gated_on_the_job_that_runs_the_suite() -> None:
"""The positive half, without which deleting `needs:` is silently green."""
scripts = vitest_scripts()
known = set(dockerfiles())
faults: list[str] = []
for rel, job_id, dockerfile, jobs in publishing_jobs():
if dockerfile not in known:
faults.append(
f"{rel}: job `{job_id}` builds `{dockerfile}`, which is not a tracked Dockerfile. "
f"The population here is derived from the git index, so this job's image is "
f"UNCHECKED — fix the path or widen `dockerfiles()`."
)
continue
if not carries_spa_source(dockerfile):
continue
gates = _transitive_needs(job_id, jobs)
runners = sorted(
gate for gate in gates if any(suite_invocations(body, scripts) for body in _run_bodies(jobs.get(gate, {})))
)
if not runners:
faults.append(
f"{rel}: job `{job_id}` publishes an image built from `{dockerfile}`, which carries "
f"the SPA source, but none of the jobs it transitively `needs:` ({sorted(gates)}) "
f"runs the SPA suite. Since ersatztv#887 the image build deliberately does not run "
f"the suite itself, so this `needs:` edge is the ONLY thing standing between a red "
f"suite and a published image."
)
assert not faults, "\n".join(faults)
def test_every_spa_suite_invocation_is_UNFILTERED() -> None:
"""One list, and it is empty. A narrowed run is the hand-maintained population coming back.
Driven off the YAML walk rather than the file text, so it reads exactly what the gate assertion
above reads. `test_the_yaml_walk_finds_every_suite_invocation_the_TEXT_does` is what stops the
walk from silently narrowing.
"""
scripts = vitest_scripts()
faults: list[str] = []
for path in workflow_files():
rel = path.relative_to(REPO_ROOT).as_posix()
jobs = _jobs(yaml.safe_load(path.read_text(encoding="utf-8")))
for job_id, job in jobs.items():
for body in _run_bodies(job):
for _, command in suite_invocations(body, scripts):
narrowing = narrowing_arguments(command, scripts)
if narrowing:
faults.append(
f"{rel}: job `{job_id}` runs the SPA suite but restricts which specs run "
f"({narrowing}): `{command.strip()}`. The gating run must be the WHOLE "
f"suite — a filtered one re-creates the hand-maintained population "
f"ersatztv#887 removed."
)
assert not faults, "\n".join(faults)
def test_the_yaml_walk_finds_every_suite_invocation_the_TEXT_does() -> None:
"""The walk and a plain text scan must agree per file, or the walk has a blind spot.
Both assertions above read `jobs.<id>.steps[].run`. A step list the walk cannot reach makes a
suite run structurally invisible while the guard still reports a clean pass, and absence of
output is not evidence. The two shapes that would escape BOTH halves — a local composite action
and a `jobs.<id>.uses:` reusable workflow — are asserted not to exist AT ALL by
`test_workflow_persist_credentials.py`, repo-wide and for the same reason; they are not
re-asserted here, because two guards on one condition mask each other (ersatztv#685).
"""
scripts = vitest_scripts()
disagreements: list[str] = []
for path in workflow_files():
text = path.read_text(encoding="utf-8")
jobs = _jobs(yaml.safe_load(text))
walked = sum(len(suite_invocations(body, scripts)) for job in jobs.values() for body in _run_bodies(job))
scanned = len(suite_invocations(text, scripts))
if walked != scanned:
disagreements.append(
f"{path.name}: the jobs.<id>.steps walk found {walked} SPA-suite invocation(s) but "
f"the text scan found {scanned}. The walk is what both assertions above run over, so "
f"the difference is suite runs this guard cannot see — widen `_run_bodies`."
)
assert not disagreements, "\n".join(disagreements)
def test_no_run_BODY_builds_or_pushes_an_image() -> None:
"""The publishing-job derivation keys on the action; assert nothing publishes around it.
Both halves would agree on zero if an image were built by a raw `docker push` in a shell body,
so this reports the shape rather than leaving it uncovered.
"""
stray: list[str] = []
for path in workflow_files():
rel = path.relative_to(REPO_ROOT).as_posix()
jobs = _jobs(yaml.safe_load(path.read_text(encoding="utf-8")))
for job_id, job in jobs.items():
for body in _run_bodies(job):
for number, command in logical_commands(body):
for segment in command_segments(command):
head = _strip_prefixes(segment)
publishes = head[:3] == ["docker", "buildx", "build"] or (
len(head) >= 2 and head[0] == "docker" and head[1] in {"push", "build"}
)
if publishes:
stray.append(f"{rel}: job `{job_id}`, line {number} of a run body: `{command.strip()}`")
assert not stray, (
f"image build/push outside `{PUBLISH_ACTION}`: {stray}. The publishing-job population above "
f"is derived from that action, so these jobs are NOT checked for the suite gate — extend "
f"`publishing_jobs()` before adding one."
)
# ------------------------------------------------------------------------------------------------
# Anti-vacuity and negative controls
# ------------------------------------------------------------------------------------------------
def test_the_populations_are_NOT_empty() -> None:
"""Every assertion above passes over an empty set, which is how a completeness guard dies."""
assert dockerfiles(), "`git ls-files` reported no Dockerfile — the derivation is broken, not the repo."
assert workflow_files(), f"`git ls-files` reported no {WORKFLOW_DIR}/*.yml — the derivation is broken."
scripts = vitest_scripts()
assert scripts, (
f"no script in {PACKAGE_JSON} was read as running vitest, so every assertion in this file "
"measures an empty set: no command can be recognised as a suite run. `direct_vitest_args` "
"stopped matching the script body, or the scripts moved."
)
spa_carrying = [rel for rel in dockerfiles() if carries_spa_source(rel)]
assert spa_carrying, (
f"none of {dockerfiles()} was read as carrying the SPA source, so "
"`test_every_IMAGE_PUBLISHING_job_is_gated_on_the_job_that_runs_the_suite` skipped every "
"job and asserted nothing. `carries_spa_source` stopped matching the `COPY web/` it derives."
)
published = publishing_jobs()
assert published, (
f"no job in {WORKFLOW_DIR} uses `{PUBLISH_ACTION}`, so the gate assertion ran over an empty "
"population. Either the repo stopped publishing images or `_canonical_action` stopped "
"recognising the action."
)
runs = [
(path.name, number)
for path in workflow_files()
for number, _ in suite_invocations(path.read_text(encoding="utf-8"), scripts)
]
assert runs, (
"no workflow runs the SPA suite at all. The image gate then rests on a job that runs "
"nothing, and `test_every_spa_suite_invocation_is_UNFILTERED` asserts over an empty set."
)
def test_the_SCRIPT_derivation_separates_vitest_from_playwright() -> None:
"""Pinned in both directions: this table is what every other assertion here keys on.
`test` in and `test:ui-e2e` out is the whole reason there is no exemption list. If the
derivation drifted to include `test:ui-e2e`, the guard would demand an unfiltered gate on a
Playwright run; if it drifted to exclude `test`, nothing in the repo would be recognised as a
suite run and every assertion above would pass vacuously.
"""
scripts = vitest_scripts()
assert "test" in scripts, scripts
assert "test:ui-e2e" not in scripts, scripts
assert "build" not in scripts and "lint" not in scripts, scripts
def test_the_detector_recognises_the_SPELLINGS_it_claims_to() -> None:
"""Negative control on the predicates, driven directly rather than through the tree.
The mutation proof drives the real Dockerfile; this covers the spellings the tree does not
contain, so a detector that quietly stopped matching one is caught without waiting for someone
to write it.
"""
scripts = {"test", "test:unit"}
for command in (
"npm test -- --run",
"npm run test",
"npm run test:unit",
"npx vitest run",
"npm exec vitest run",
"vitest run",
"cd web && npm test -- --run",
"npm run lint && npm test -- --run && npm run build",
"RUN npm test -- --run",
"env CI=1 npm test -- --run",
# The one-line YAML step form, which roughly half this repo's `run:` steps use.
"run: npm test -- --run",
"- run: npm test -- --run",
):
assert suite_invocations(command, scripts), command
for command in (
"dotnet test --configuration Release",
"npm ci",
"npm run build",
"npm run lint",
"npm run typecheck",
"npm run test:ui-e2e", # not in `scripts`, i.e. not a vitest script
"playwright test",
# Prose ABOUT the suite is not a run of it. The first version of this guard reported both of
# these, which is a false red in exactly the files that explain the rule.
"echo the vitest suite does not run here",
"echo skipping npm test in this stage",
# A non-RUN Dockerfile instruction does not execute during a build.
"CMD npm test -- --run",
"COPY vitest.config.ts ./",
):
assert not suite_invocations(command, scripts), command
assert narrowing_arguments("npm test -- --run", scripts) == []
assert narrowing_arguments("npm test -- --run --reporter verbose", scripts) == []
assert narrowing_arguments("npm test -- --run --exclude 'src/api/x.guard.test.ts'", scripts) == [
"--exclude",
"src/api/x.guard.test.ts",
]
assert narrowing_arguments("npm test -- --run --project=spa", scripts) == ["--project=spa"]
assert narrowing_arguments("npm test -- --run src/api", scripts) == ["src/api"]
assert narrowing_arguments("npm test -- --run --changed", scripts) == ["--changed"]
def test_a_COMMENT_is_inert_but_only_a_LEADING_hash_is() -> None:
"""`docker/Dockerfile` names the rule it enforces; that must not trip the rule.
The second half is the one that matters: a `#` mid-line does not make what precedes it a
comment, and treating it as one would let a real invocation hide behind a trailing note.
"""
scripts = {"test"}
assert not suite_invocations("# npm test -- --run\n", scripts)
assert not suite_invocations(" # the vitest suite is not run in this stage\n", scripts)
assert suite_invocations("RUN npm test -- --run # left over from before\n", scripts)
def test_the_continuation_JOIN_keeps_flags_with_their_command() -> None:
"""A multi-line invocation is one command; its flags must not read as separate lines.
This is the exact shape the removed exclusion pair had, so a guard that scanned raw lines would
see `npm test -- --run` as unfiltered and report it clean.
"""
scripts = {"test"}
multiline = "RUN npm test -- --run \\\n --exclude 'src/api/a.test.ts' \\\n --exclude 'b.test.ts'\n"
found = suite_invocations(multiline, scripts)
assert len(found) == 1, found
assert narrowing_arguments(found[0][1], scripts) == ["--exclude", "src/api/a.test.ts", "--exclude", "b.test.ts"]
def test_the_spa_carrying_predicate_separates_the_two_real_DOCKERFILES() -> None:
"""Pinned both ways, because the gate assertion is SKIPPED for everything it answers False for.
A predicate that drifted to False for `docker/Dockerfile` would make the positive half vacuous
while `test_the_populations_are_NOT_empty` still passed on some other file.
"""
assert carries_spa_source("docker/Dockerfile")
assert not carries_spa_source("docker/ci/Dockerfile")