#!/usr/bin/env bash # Prove — at runtime, every run — that the delimiter ban actually DETECTS a delimiter (ersatztv#767). # # WHY THIS EXISTS AS A SHELL STEP RATHER THAN A TEST. The `scan` job's guarantee used to be a pytest # that asserted the ban command works. DEMONSTRATED: ONE new file at the repo root disarms the whole gate — a # `pytest.ini` carrying `addopts = -k "not delimiter_banned"`, or a `conftest.py` with # `pytest_collection_modifyitems` — which deselects the ban test AND the tests guarding it, leaving # every job green while a delimiter sits in `build`'s `Smoke` body and the image publishes. That tier # sits above every `env:` a test can reconstruct, and above the tests themselves: anything living # inside pytest can be deselected by pytest's own configuration. So the positive control lives here. # # IT RUNS IN THE REAL CHECKOUT, NOT A COPY, and that is the whole point. An isolated copy does not # inherit the repo-root `pytest.ini`/`conftest.py`/`pyproject.toml` that a disarm would live in, so a # copy-based proof reports the ban healthy while the job's actual invocation is # deselected — measured. Same cwd, same rootdir, same config and # the same BAN INVOCATION as the run being vouched for — not the same command line, since the # workflow step also runs the wiring tests and this runs only the ban file. Only the workflow file # differs, and only for a moment. # # The workflow file is restored by an EXIT trap. That covers a normal exit and SIGTERM (measured: # `git status` clean afterwards); it does NOT cover SIGKILL — a cancelled or OOM-killed job leaves # the checkout poisoned. Do not upgrade this to "cannot leave a poisoned tree behind": it can. # It stays harmless for a reason worth stating rather than assuming. The workspace that could be # reused is THIS job's own, on the persistent `small` lane (`scan` has no `container:` — see the # carve-out in `ci.required-job-step-execution-markers`), not `build`'s, which checks out fresh in a # separate job. A leftover poisoned workflow there makes the NEXT `scan` red rather than quiet, and # `actions/checkout` restores it anyway. Fail-closed in both directions. # # The failure direction is the safe one throughout: if this script cannot do its work (no python3, a # moved anchor, an unwritable tree) it exits non-zero and the release is blocked, never silently # skipped — `set -euo pipefail` extends that to the unexpected cases. # # AND IT ONLY VOUCHES FOR THE EXACT EXPECTED FAILURE. Three outcomes are distinguished below: pytest # passing (the ban is not enforcing), pytest failing for a NON-test reason (collection error, nothing # collected — cannot prove anything), and pytest failing on some OTHER test while the ban test itself # was deselected. Only the ban test's `[build]` parametrisation failing counts as proof. Both weaker # readings have been live bugs here: "non-zero means it noticed" reported OK on a # collection error, and the other still fell through to OK when an unrelated test reddened. # # NOTE ON THE OPENER: this file must never contain the literal two-character expression opener, both # because `scan`'s own run bodies are asserted delimiter-free and because a file containing it is the # very hazard under test. It is CONSTRUCTED from its parts below. set -euo pipefail BAN_TEST="scripts/tests/test_ci_dropped_step_guard.py" WORKFLOW=".gitea/workflows/docker-build.yml" # The line the poison is injected above, inside `build`'s `Smoke + IPTV E2E` body. Kept in one place # so that rewriting that step fails loudly here instead of silently poisoning nothing. ANCHOR='IMG="${IMAGE}:${SMOKE_SHORT_SHA}"' cd "${GITHUB_WORKSPACE:-$(pwd)}" backup="$(mktemp)" cp "$WORKFLOW" "$backup" restore() { cp "$backup" "$WORKFLOW"; rm -f "$backup"; } trap restore EXIT python3 - "$WORKFLOW" "$ANCHOR" <<'PY' import sys from pathlib import Path path, anchor = Path(sys.argv[1]), sys.argv[2] text = path.read_text() if anchor not in text: sys.exit( "::error::ci-prove-ban-detects.sh could not find its anchor in the workflow. The Smoke step " "was rewritten without updating this script, so the proof would poison nothing and pass " "vacuously. Refusing." ) # Built from parts on purpose: the literal opener must not appear in this repo's own scripts. opener = chr(36) + "{" + "{" path.write_text(text.replace(anchor, f"# {opener} steps.meta.outputs.short }}}}\n {anchor}", 1)) PY echo "Proving the delimiter ban detects an injected opener (ersatztv#767)…" output="$(mktemp)" trap 'restore; rm -f "$output"' EXIT set +e PYTHONPATH=. python3 -m pytest "$BAN_TEST" -q >"$output" 2>&1 detected=$? set -e # THREE OUTCOMES, EXHAUSTIVELY, and only the last one is a pass. "Non-zero means it noticed" is not # good enough, and neither is "exit 1 means the ban test failed": both fall through to # OK when pytest exits 1 for a DIFFERENT test's failure while the ban test was # deselected. Requiring the exact expected failure is what makes this a proof rather than a liveness # check. (`[build]` because the ban is parametrised per job, and `build` is the one whose dropped # `Smoke` publishes an unsmoked image.) EXPECTED='test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body\[build\]' if [ "$detected" -eq 0 ]; then echo "::error::THE DELIMITER BAN IS NOT ENFORCING. ${BAN_TEST} PASSED on a tree whose build/Smoke body contains an expression delimiter, so the release-path gate would let an unsmoked candidate publish. Causes seen before: a repo-root pytest.ini/conftest.py/pyproject.toml deselecting it, a neutered opener regex, 'build' dropped from DELIMITER_BAN_JOBS, or the test renamed. This check runs in the real checkout and outside pytest on purpose — anything inside pytest can be deselected by pytest's own configuration." >&2 sed 's/^/ /' "$output" >&2 exit 1 elif [ "$detected" -ne 1 ]; then echo "::error::ci-prove-ban-detects.sh could not prove anything: pytest exited ${detected} on the poisoned tree, which is not a test failure (2=collection error, 3=internal, 4=usage, 5=nothing collected). The ban may well be fine, but this check cannot vouch for it, so it refuses rather than reporting a pass it did not earn." >&2 sed 's/^/ /' "$output" >&2 exit 1 elif ! grep -qE "$EXPECTED" "$output"; then echo "::error::ci-prove-ban-detects.sh saw a test failure on the poisoned tree, but NOT the expected one. The ban test's build parametrisation did not fail, so something else did — the ban itself may be deselected or renamed while an unrelated test reddens. Refusing to read that as proof." >&2 sed 's/^/ /' "$output" >&2 exit 1 fi echo "OK: the ban detected the injected delimiter in build/Smoke (pytest exit 1, build parametrisation failed) — the gate is enforcing."