ersatztv/scripts/ci-prove-ban-detects.sh

#!/usr/bin/env bash
# Prove — at runtime, every run — that the delimiter ban actually DETECTS a delimiter (ersatztv#767).
#
# WHY THIS EXISTS AS A SHELL STEP RATHER THAN A TEST. The `scan` job's guarantee used to be a pytest
# that asserted the ban command works. DEMONSTRATED: ONE new file at the repo root disarms the whole gate — a
# `pytest.ini` carrying `addopts = -k "not delimiter_banned"`, or a `conftest.py` with
# `pytest_collection_modifyitems` — which deselects the ban test AND the tests guarding it, leaving
# every job green while a delimiter sits in `build`'s `Smoke` body and the image publishes. That tier
# sits above every `env:` a test can reconstruct, and above the tests themselves: anything living
# inside pytest can be deselected by pytest's own configuration. So the positive control lives here.
#
# IT RUNS IN THE REAL CHECKOUT, NOT A COPY, and that is the whole point. An isolated copy does not
# inherit the repo-root `pytest.ini`/`conftest.py`/`pyproject.toml` that a disarm would live in, so a
# copy-based proof reports the ban healthy while the job's actual invocation is
# deselected — measured. Same cwd, same rootdir, same config and
# the same BAN INVOCATION as the run being vouched for — not the same command line, since the
# workflow step also runs the wiring tests and this runs only the ban file. Only the workflow file
# differs, and only for a moment.
#
# The workflow file is restored by an EXIT trap. That covers a normal exit and SIGTERM (measured:
# `git status` clean afterwards); it does NOT cover SIGKILL — a cancelled or OOM-killed job leaves
# the checkout poisoned. Do not upgrade this to "cannot leave a poisoned tree behind": it can.
# It stays harmless for a reason worth stating rather than assuming. The workspace that could be
# reused is THIS job's own, on the persistent `small` lane (`scan` has no `container:` — see the
# carve-out in `ci.required-job-step-execution-markers`), not `build`'s, which checks out fresh in a
# separate job. A leftover poisoned workflow there makes the NEXT `scan` red rather than quiet, and
# `actions/checkout` restores it anyway. Fail-closed in both directions.
#
# The failure direction is the safe one throughout: if this script cannot do its work (no python3, a
# moved anchor, an unwritable tree) it exits non-zero and the release is blocked, never silently
# skipped — `set -euo pipefail` extends that to the unexpected cases.
#
# AND IT ONLY VOUCHES FOR THE EXACT EXPECTED FAILURE. Three outcomes are distinguished below: pytest
# passing (the ban is not enforcing), pytest failing for a NON-test reason (collection error, nothing
# collected — cannot prove anything), and pytest failing on some OTHER test while the ban test itself
# was deselected. Only the ban test's `[build]` parametrisation failing counts as proof. Both weaker
# readings have been live bugs here: "non-zero means it noticed" reported OK on a
# collection error, and the other still fell through to OK when an unrelated test reddened.
#
# NOTE ON THE OPENER: this file must never contain the literal two-character expression opener, both
# because `scan`'s own run bodies are asserted delimiter-free and because a file containing it is the
# very hazard under test. It is CONSTRUCTED from its parts below.
set -euo pipefail

BAN_TEST="scripts/tests/test_ci_dropped_step_guard.py"
WORKFLOW=".gitea/workflows/docker-build.yml"
# The line the poison is injected above, inside `build`'s `Smoke + IPTV E2E` body. Kept in one place
# so that rewriting that step fails loudly here instead of silently poisoning nothing.
ANCHOR='IMG="${IMAGE}:${SMOKE_SHORT_SHA}"'

cd "${GITHUB_WORKSPACE:-$(pwd)}"

backup="$(mktemp)"
cp "$WORKFLOW" "$backup"
restore() { cp "$backup" "$WORKFLOW"; rm -f "$backup"; }
trap restore EXIT

python3 - "$WORKFLOW" "$ANCHOR" <<'PY'
import sys
from pathlib import Path

path, anchor = Path(sys.argv[1]), sys.argv[2]
text = path.read_text()
if anchor not in text:
    sys.exit(
        "::error::ci-prove-ban-detects.sh could not find its anchor in the workflow. The Smoke step "
        "was rewritten without updating this script, so the proof would poison nothing and pass "
        "vacuously. Refusing."
    )
# Built from parts on purpose: the literal opener must not appear in this repo's own scripts.
opener = chr(36) + "{" + "{"
path.write_text(text.replace(anchor, f"# {opener} steps.meta.outputs.short }}}}\n          {anchor}", 1))
PY

echo "Proving the delimiter ban detects an injected opener (ersatztv#767)…"
output="$(mktemp)"
trap 'restore; rm -f "$output"' EXIT
set +e
PYTHONPATH=. python3 -m pytest "$BAN_TEST" -q >"$output" 2>&1
detected=$?
set -e

# THREE OUTCOMES, EXHAUSTIVELY, and only the last one is a pass. "Non-zero means it noticed" is not
# good enough, and neither is "exit 1 means the ban test failed": both fall through to
# OK when pytest exits 1 for a DIFFERENT test's failure while the ban test was
# deselected. Requiring the exact expected failure is what makes this a proof rather than a liveness
# check. (`[build]` because the ban is parametrised per job, and `build` is the one whose dropped
# `Smoke` publishes an unsmoked image.)
EXPECTED='test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body\[build\]'

if [ "$detected" -eq 0 ]; then
    echo "::error::THE DELIMITER BAN IS NOT ENFORCING. ${BAN_TEST} PASSED on a tree whose build/Smoke body contains an expression delimiter, so the release-path gate would let an unsmoked candidate publish. Causes seen before: a repo-root pytest.ini/conftest.py/pyproject.toml deselecting it, a neutered opener regex, 'build' dropped from DELIMITER_BAN_JOBS, or the test renamed. This check runs in the real checkout and outside pytest on purpose — anything inside pytest can be deselected by pytest's own configuration." >&2
    sed 's/^/  /' "$output" >&2
    exit 1
elif [ "$detected" -ne 1 ]; then
    echo "::error::ci-prove-ban-detects.sh could not prove anything: pytest exited ${detected} on the poisoned tree, which is not a test failure (2=collection error, 3=internal, 4=usage, 5=nothing collected). The ban may well be fine, but this check cannot vouch for it, so it refuses rather than reporting a pass it did not earn." >&2
    sed 's/^/  /' "$output" >&2
    exit 1
elif ! grep -qE "$EXPECTED" "$output"; then
    echo "::error::ci-prove-ban-detects.sh saw a test failure on the poisoned tree, but NOT the expected one. The ban test's build parametrisation did not fail, so something else did — the ban itself may be deselected or renamed while an unrelated test reddens. Refusing to read that as proof." >&2
    sed 's/^/  /' "$output" >&2
    exit 1
fi
echo "OK: the ban detected the injected delimiter in build/Smoke (pytest exit 1, build parametrisation failed) — the gate is enforcing."