diff --git a/.gitea/workflows/docker-build.yml b/.gitea/workflows/docker-build.yml index 04508546f..f45e46056 100644 --- a/.gitea/workflows/docker-build.yml +++ b/.gitea/workflows/docker-build.yml @@ -41,6 +41,17 @@ name: Build ErsatzTV Image # (verified, throwaway PR #418) and we don't rely on how branch protection treats a skipped # REQUIRED context. See docs/ci-cd.md -> "Docs-only skip". # +# RELEASE-PATH DELIMITER GATE (ersatztv#767): the `scan` job runs the PyYAML-based delimiter-ban +# test and is a `needs:` of `build`, so a `${{` opener in a banned job's `run:` body means `build` +# never runs. It is deliberately NOT gated by either skip below: the gate's coverage must not depend +# on a detector the gate is not allowed to trust, and it is cheap enough that gating it buys nothing. +# (Do NOT justify that with "the docs-only path still builds an image" — it does not. `Build and +# push` carries the docs_only gate too; a tag build is unaffected only because the script forces +# docs_only=false there.) Note it installs from PyPI (setup-python + pip), putting a NEW network +# dependency between a `v*` tag and its image. Not the only one on this path — `test` runs +# `dotnet restore` and `npm ci` behind actions/cache, and a cache miss reaches nuget.org/npm — but +# newly added here. Fail-closed and loud, and still a real availability dependency. +# # ALREADY-VALIDATED SKIP (ersatztv#420): a second, sibling gate in `test`, `migrations` and # `functional-e2e` only (NOT `build`). On a push-to-main merge commit, `id: revalidate` runs # `scripts/ci-detect-already-validated.sh`, which emits `skip=true` only when the merged tree is @@ -643,6 +654,106 @@ jobs: # server. Its exit status is Playwright's. scripts/e2e-ui.sh + # THE DELIMITER BAN, RE-CHECKED ON THE RELEASE PATH ITSELF (ersatztv#767). + # + # The ban that keeps `build`'s `Smoke + IPTV E2E` from being silently dropped was enforced only by + # `test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body` in the + # `script-tests` job of pr-checks.yml — `on: pull_request`, and NOT a required context. So the ban + # was REVIEW-TIME only: nothing re-checked it on a `v*` tag push, which is precisely when the + # candidate image is published and `DeployStack jazz-media` promotes it. + # + # WHY A JOB AND NOT A STEP INSIDE `build`. A step cannot protect the thing it shares a job with: + # `build` is what publishes, so a guard step there fails OPEN if the runner drops it, and "my body + # has no opener so I cannot be dropped" is circular when the only thing enforcing that property is + # the same PR-only test being backstopped. As a `needs:` of `build`, a red here means `build` never + # runs at all — the image is not built, let alone pushed. Fail-closed by dependency, not by + # assertion. + # + # WHY IT RUNS THE REAL PYTEST rather than a bespoke scanner. The first cut of #767 hand-parsed the + # workflow YAML in stdlib Python, to avoid provisioning PyYAML on `build`'s bare runner. Two + # independent reviews found ~10 false NEGATIVES in that parser within one round (flow mappings + # `{run: …}`, a quoted `"run":` key, aliases, multiline quoted scalars) — i.e. it was strictly + # WEAKER than the check it was meant to backstop, in the one direction that matters for a security + # gate. Running the existing PyYAML-based test needs no second implementation of "what is a `run:` + # body" and therefore has no drift surface. `small` is git-only, so Python is provisioned here the + # same way `script-tests` does it. + # + # This job's OWN steps carry #756 markers and a trailing assert, so a drop inside THIS job is + # caught too. That terminates the regress at the same axiom the sibling guards already rest on — + # to fail open you must now drop the pytest step AND the assert step, rather than either one. + # + # THIS PUTS A `small`-LANE JOB BACK ON THE TAG PATH, which ersatztv#535 deliberately moved away + # from — say so rather than letting it look accidental. #535 split the git-only gates into + # pr-checks.yml because on the v26.12.0 tag they wedged in act's setup phase, were killed, and + # reported `failure` with no logs. The blast radius here is WORSE than it was then: as a `needs:` + # of `build`, that flake would not merely redden a status, it would skip the build and produce no + # release image at all. + # + # It is acceptable now for a stated reason rather than an assumed one, and the evidence is weaker + # than it first looks — so read the limits. Per `ci.small-lane-git-only`, the lane's per-job cap was + # forced to 10g by its two HEAVIEST members (this file's `build` AND ci-image.yml's toolchain + # buildx), not by `build` alone, and that cap is what pinned the lane to one slot on a 25 GiB host; + # both were moved off in server-management#639, after which the lane is git-only and runs wide and + # tiny. What has NOT been demonstrated is this lane on a TAG PUSH: `script-tests` runs there happily + # but lives in pr-checks.yml (`on: pull_request`), so it has never exercised the condition #535 + # measured, and #767's own runs (1928/1929) were `workflow_dispatch` on a scratch branch. The + # lane-width argument is what carries this, not a like-for-like observation. If the wedging returns, + # move this job to `ubuntu-latest` rather than weakening the `needs:` edge — a slower gate is fine, + # an optional one is not. + scan: + name: Delimiter ban (release path) + runs-on: small + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install test dependencies + run: | + "${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark deps + python3 -m pip install --disable-pip-version-check --quiet pytest pyyaml + # The ban test plus the structural tests that hold this job's own shape. NOT the whole + # scripts/tests suite: that is `script-tests`'s job, it needs jq/git preflights, and an + # unrelated pytest regression must not be able to block a release. + - name: Run the delimiter-ban tests + run: | + "${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark ban + PYTHONPATH=. python3 -m pytest scripts/tests/test_ci_dropped_step_guard.py scripts/tests/test_ci_release_path_scan_job.py -q + # THE POSITIVE CONTROL, and it is deliberately NOT a test (ersatztv#767). The step above proves + # the ban HOLDS; it cannot prove the ban would NOTICE. Review disarmed the entire gate with one + # repo-root `pytest.ini` (`addopts = -k "not delimiter_banned"`) or `conftest.py` + # (`pytest_collection_modifyitems`), which deselects the ban test and every test guarding it, + # leaving all jobs green with a delimiter sitting in `Smoke`. Nothing inside pytest can be + # trusted to catch that, because pytest's own configuration outranks it. + # + # So this poisons the checked-out workflow, re-runs the SAME command, and fails the job if it + # PASSES. It runs in the real checkout — an isolated copy does not inherit the repo-root config + # a disarm would live in, which made the first version of this script report healthy while the + # job's real invocation was deselected. The workflow file is restored by an EXIT trap. + - name: Prove the ban would DETECT a delimiter (ersatztv#767) + run: | + "${GITHUB_WORKSPACE:-.}/scripts/ci-step-ran.sh" mark selfcheck + scripts/ci-prove-ban-detects.sh + # No `if:` — see the sibling guards in `test`/`migrations` for why the default `success()` is + # the wanted condition. Both keys are `--always`: every step in this job is unconditional. + # + # THE MARKER-PATH RATIONALE DOES NOT TRANSFER HERE, and assuming it did would be the mistake + # `ci.required-job-step-execution-markers` itself warns about. That record says the run-id and + # attempt keying is "defence in depth" because "these jobs get a fresh container, which is the + # primary protection". This job has NO `container:` — it is on `small`, where RUNNER_TEMP is + # the shared host /tmp. So here the keying is the ONLY protection, and the residual is a + # single-job re-run that does not increment GITHUB_RUN_ATTEMPT: it would find the previous + # attempt's marker file and the assert would pass even had the pytest step been dropped. + # Identity was read off a real run rather than assumed — run 1929 printed + # `Marker identity: job=scan run=1929 attempt=1 (from the runner)`, so all three variables are + # populated on this lane. + - name: Assert every expected step executed (ersatztv#756) + run: >- + scripts/ci-step-ran.sh assert + --always deps ban selfcheck + build: name: Build & push image (amd64) # Moved back off `small` (server-management#639). This is the one HEAVY job that @@ -654,12 +765,15 @@ jobs: # # The `ubuntu-latest` queueing that sent it to `small` in the first place # (server-management#574: a PR-run skip stuck 31 min behind long builds) does not - # come back, because `needs: [test, migrations]` means this job cannot be - # dispatched until those two have already finished — by which point the lane it + # come back, because `needs: [test, migrations, scan]` means this job cannot be + # dispatched until those three have already finished — by which point the lane it # was queueing behind has drained. Real builds (main/tags) get the full # ubuntu-latest allotment: 4 CPUs / 10g on ci-runner (.127). runs-on: ubuntu-latest - needs: [test, migrations] + # `scan` (ersatztv#767) re-checks the delimiter ban on the release path. As a `needs:` its red + # SKIPS this job outright, so a delimiter in `Smoke + IPTV E2E` can no longer reach the point + # where an image is published and never booted. + needs: [test, migrations, scan] if: github.event_name != 'pull_request' steps: - name: Checkout @@ -740,15 +854,15 @@ jobs: # body delimiter-free the class is unreachable here — held by # test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body. # - # This job gets the ban but NOT (yet) the per-step markers — and NOT because "it is not a - # required context, so the markers would buy little". That is the same family as the - # "smaller cost (no required context lies)" dismissal that - # `ci.required-job-step-execution-markers` records as FALSE about this very step. Markers - # here would buy strictly MORE than the ban does: the ban is enforced only by `script-tests`, - # which is `on: pull_request` and not a required check, so nothing re-checks it when a release - # is actually cut, whereas markers would be a runtime, fail-closed gate on the tag path itself. - # It is deferred because this step's `if:` carries a publish-ref condition the guard's - # always/gated buckets do not model. Tracked as ersatztv#767. + # The ban IS re-checked on the release path now (ersatztv#767): the `scan` job above runs the + # PyYAML-based ban test and is a `needs:` of this job, so a delimiter here means `build` never + # runs and no image is published. Do not re-add the note that once stood here saying the ban is + # "review-time only, tracked as #767" — that was true before the `scan` job existed. + # + # This step still carries no per-step markers, and that is a genuine (smaller) residual rather + # than a dismissal: markers would additionally catch a drop caused by something OTHER than a + # delimiter. Adding them needs a bucket modelling this step's publish-ref `if:`, which the + # guard's always/gated buckets do not express. The delimiter class itself is covered. - name: Smoke + IPTV E2E (assert key endpoints) if: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && steps.detect.outputs.docs_only != 'true' }} env: diff --git a/docs/ci-cd.md b/docs/ci-cd.md index ccc0f69f7..d3b38eda7 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -202,7 +202,12 @@ rather than in `docker-build.yml` — see that section (ersatztv#535). the lane is a checkout plus a `git diff`: `decisions-guard`, `ci-image-pin`, `docs-reminder` — plus `script-tests`, which is a checkout plus a `pytest` run needing only `pytest` and `pyyaml` (ersatztv#631; it is NOT stdlib-only — that assumption is what turned the -job red on its first CI run, see below). Nothing there runs a compiler or a `docker build`, which is why the lane +job red on its first CI run, see below) — plus **`scan`** (ersatztv#767), the same +lightweight-Python shape. `scan` is the lane member to think hardest about before changing anything +here: it is the only one that lives in `docker-build.yml` rather than `pr-checks.yml`, so the only +one that runs on a **tag push**, and the only one whose failure does not merely redden a status but +**skips `build`** — an OOM or a wedge there yields no release image at all. +Nothing there runs a compiler or a `docker build`, which is why the lane can be capped at 1 GiB per job. The lightweight-Python jobs are the deliberate edge of the "git-only" rule, not an exception to it: `setup-python` + `pip install pytest` + a suite whose heaviest allocation is a handful of temp-dir git repos stays far under the cap. Route a heavy job here and it will OOM — give it @@ -260,7 +265,7 @@ thing that ran in the lane) — and their 10 GiB requirement set the lane's per- on a 25 GiB host permits exactly one slot. Moving both to `ubuntu-latest` made the lane genuinely tiny, so it could widen to **4 slots across two hosts while committing less RAM to CI than the single slot did**. `docker-build.yml`'s `build` does not re-create #574's -skip-task queueing, because `needs: [test, migrations]` means it cannot be dispatched until +skip-task queueing, because `needs: [test, migrations, scan]` means it cannot be dispatched until the lane it would queue behind has already drained. ### CI build memory: no persistent compiler servers (ersatztv#406) @@ -647,23 +652,93 @@ Two independent mechanisms hold it, and neither is redundant: jazz-media` promotes exactly that image. Its two payloads moved into the step's `env:`, so the ban cost nothing. - **Be precise about what that buys, because it is less than it looks.** The ban is enforced by the - `script-tests` job, which lives in `pr-checks.yml` (`on: pull_request`) and is **not** a required - context. So it is a *review-time* check on the PR that would introduce a delimiter — not a - fail-closed gate on the release path. `pr-checks.yml` does not run on a `v*` tag push at all, and - `build` has no markers and no guard, so a delimiter that ever reached `main` would still drop - `Smoke` on the tag build and go green. `main` being PR-only (#743) means such a change must pass - through a PR where `script-tests` reddens, but a red on a non-required check does not block the - merge server-side. Strictly better than before, not closed. Tracked as **ersatztv#767**. + **The ban is re-checked on the release path itself (ersatztv#767).** It used to be enforced only by + the `script-tests` job, which lives in `pr-checks.yml` (`on: pull_request`) and is **not** a + required context — a *review-time* check on the PR that would introduce a delimiter, not a gate on + the release. `pr-checks.yml` does not run on a `v*` tag push at all, so a delimiter that ever + reached `main` would still drop `Smoke` on the tag build and go green; `main` being PR-only (#743) + meant such a change had to pass through a PR where `script-tests` reddens, but a red on a + non-required check does not block the merge server-side. - The reason it was deferred is that `Smoke`'s `if:` carries a publish-ref condition the guard's - always/gated buckets do not model — that is the whole of it. An earlier draft here also claimed - measuring such a guard "would mean cutting a real release tag", and that was simply **wrong**: - `build` runs on every push to `main` (`if: github.event_name != 'pull_request'`) and `Smoke`'s ref - condition admits `refs/heads/main`, so the runs-side is exercised by any ordinary merge — including - the one that lands this change — and the legitimately-skipped side by a `workflow_dispatch` on any - other ref. Left recorded because a false cost estimate is exactly what talks the next person out of - a tracked follow-up. `functional-e2e` is delimiter-free too but is deliberately **not** banned: it is + There is now a **`scan` job** (`Delimiter ban (release path)`) that runs the PyYAML-based ban test, + and **`build` lists it in `needs:`**. That single edge is the fail-closed property: a red `scan` + means `build` is skipped outright, so the image is never built, let alone pushed. + + **Why a job and not a step inside `build`.** A step cannot protect the job it lives in. `build` is + what publishes, so a guard step there fails **open** if the runner drops it — and the defence + ("the guard's own body has no opener, so it cannot be dropped") is circular when the only thing + enforcing that property is the same PR-only test being backstopped. This was the first design and + two independent reviews rejected it for exactly that. + + **Why it runs the real pytest and not a bespoke scanner.** The same first cut hand-parsed the + workflow YAML in stdlib Python, to avoid provisioning PyYAML on `build`'s bare runner. Review found + ~10 **false negatives** in that parser in one round — flow mappings (`{run: …}`), a quoted + `"run":` key, aliases, multiline quoted scalars — making it strictly *weaker* than the check it + backstopped, in the only direction that matters for a security gate. Running the existing test + needs no second definition of "what is a `run:` body", so it has no drift surface at all. `scan` + runs on `small` and provisions Python the same way `script-tests` does. + + The wiring is held by `scripts/tests/test_ci_release_path_scan_job.py` — `build` depends on it, it + carries **no job-level `if:`** (one that excluded the tag push would restore the hole; one that + skipped the job would skip `build` too), no step is `continue-on-error`, and every one of its own + `run:` bodies is delimiter-free. Its steps also carry #756 markers and a trailing assert, so a drop + *inside this job* is caught as well. + + The gate's guarantee is one **execution probe**, not a set of assertions about the command: it runs + the scan step's real `run:` body — with the workflow, job and step `env:` tiers layered as the + runner would — against a copy of the repo whose `Smoke` body carries an injected delimiter, and + requires a non-zero exit, with a clean-tree negative control. Shape assertions were tried twice and + lost twice (nine disarms across two rounds, from `echo`ing the command to `PYTEST_ADDOPTS` one env + tier up), so **do not replace the probe with cheaper checks about the command.** One tier is + unreachable by any static reconstruction — a step writing to `$GITHUB_ENV` injects into later steps + at runtime — and is therefore **banned** in this job rather than modelled. + + **And one check deliberately is not a test at all.** The tests above prove the ban *holds*; they + cannot prove it would *notice*, because pytest's own configuration outranks them — a repo-root + `pytest.ini` with `addopts = -k "not delimiter_banned"`, or a `conftest.py` with + `pytest_collection_modifyitems`, deselects the ban test and every test guarding it, leaving the + whole suite green with a delimiter sitting in `Smoke`. So `scan`'s last working step runs + **`scripts/ci-prove-ban-detects.sh`**, which poisons the checked-out workflow, re-runs the ban + test, and requires **the ban test's `build` parametrisation to be the failure**. Not "the same + command" (the step also runs the wiring tests; the script runs only the ban file) and not merely + "fails if it passes": any other outcome — exit 5 because everything was deselected, exit 2 on a + collection error, or a failure in some unrelated test — is a **refusal**, not a pass. That + exhaustiveness is the load-bearing part, because each weaker reading was a live bug in an earlier + draft: "non-zero means it noticed" vouched for the gate on a collection error, and the deselection + disarm this script exists to catch exits **5**, not 1. + + It runs in the **real checkout** — an isolated copy does not inherit the repo-root config a disarm + would live in, which is exactly how the first version of that script reported healthy while the + real invocation was deselected. The workflow file is + restored by an `EXIT` trap, and the step carries a `selfcheck` marker so dropping it reddens the + job. + + Two further tests exist because review disarmed the gate in ways an obvious test misses. **The + pytest invocation is checked as a command, not as a substring**: the ban test's path must appear as + a bare token on the real `python3 -m pytest` line, that line may carry none of + `-k`/`-m`/`--deselect`/`--ignore`/`--collect-only`, and its exit status may not be swallowed by + `||`/`&&`/`;`/`|`. Commenting the invocation out and echoing instead, or appending + `-k 'not delimiter_banned'`, each left the suite green with the release path unguarded. And **the + ban's SCOPE is pinned** — `DELIMITER_BAN_JOBS` must still contain `build`, because dropping that + one literal makes `scan` pass while checking nothing about the job that publishes the image. + + Two operational notes. `scan` is a `small`-lane job on the tag path, which ersatztv#535 moved away + from after such jobs wedged in act's setup phase on the v26.12.0 tag; that is safe again only + because server-management#639 took `build` off the lane. If it ever returns, move `scan` to + `ubuntu-latest` — never weaken the `needs:` edge. And `scan` installs from PyPI, so a network blip + now sits between a `v*` tag and its image; fail-closed and loud, but real. + + What this does **not** claim: that no step can ever fail to run for a reason other than the + interpolation drop. It moves the terminal assumption — to fail open you must now drop the pytest + step **and** the assert step, rather than either one alone. + + Measuring a guard on this path does **not** require cutting a release, and an earlier draft here + claiming it would was simply wrong: `build` runs on every push to `main` + (`if: github.event_name != 'pull_request'`), and a `workflow_dispatch` on any other ref runs the + job while `Build and push` publishes nothing (its `push:` is gated on `main`/`v*`). That is how + #767 was verified — see the decision record for the run ids. + + `functional-e2e` is delimiter-free too but is deliberately **not** banned: it is advisory by declaration, and the rule is "ban where a drop is consequential", not "ban wherever it is currently free". `api-docs` and `format` keep one `github.base_ref` each in a detect step and gate nothing that ships. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index a9a3def24..16d02559d 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -56,7 +56,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `ci.monitor-armed-at-pr-open` | Arm a CI monitor on the PR head sha the moment the PR opens, polling the commit-status endpoint — not at the end of the work. | 2026-07-21 | [link](records/ci/monitor-armed-at-pr-open.md) | | `ci.no-host-health-gating` | Push when your work is validated — never SSH to bumblebee to sample load/RAM first, and never hand-schedule around other sessions' runs. | 2026-07-21 | [link](records/ci/no-host-health-gating.md) | | `ci.peak-anon-measurement` | The `test` job's headline memory figure is a sampled high-water mark of cgroup `anon`, produced by `scripts/ci-peak-anon.sh`; `memory.peak` and the end-of-job `anon`/`file` split are kept only as a cache-inflated reference. | 2026-07-19 | [link](records/ci/peak-anon-measurement.md) | -| `ci.required-job-step-execution-markers` | A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`'s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark ` as its FIRST act, and the job's LAST step calls `ci-step-ran.sh assert --always --gated `, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step's `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. | 2026-08-10 | [link](records/ci/required-job-step-execution-markers.md) | +| `ci.required-job-step-execution-markers` | A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`'s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark ` as its FIRST act, and the job's LAST step calls `ci-step-ran.sh assert --always --gated `, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step's `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. The ban is enforced on the RELEASE PATH itself, not only in review (#767): a `scan` job runs the PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and no image is published. A guard STEP inside `build` was tried first and is wrong — a step cannot protect the job it publishes from, and "my body has no opener so I cannot be dropped" is circular when only the PR-only test enforces that. The pytest in `script-tests` remains, but it is `on: pull_request` and not a required context, so it alone left the tag path unchecked. | 2026-08-10 | [link](records/ci/required-job-step-execution-markers.md) | | `ci.root-screenshot-guard` | The Husky `pre-commit` hook refuses a staged root-level `*.png` (belt-and-suspenders with the `.gitignore` rule); nested `*.png` real assets are unaffected. | 2026-07-12 | [link](records/ci/root-screenshot-guard.md) | | `ci.runner-placement` | No persistent Roslyn compiler server survives a CI build (`UseSharedCompilation=false` etc., runner env + Dockerfile `ENV`); every `services:` container gets its own explicit `--memory`/`--memory-swap`/`--cpus` cap (it does not inherit the job container's). | 2026-07-17 | [link](records/ci/runner-placement.md) | | `ci.script-tests-job` | The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it. | 2026-07-26 | [link](records/ci/script-tests-job.md) | diff --git a/docs/decisions/records/ci/required-job-step-execution-markers.md b/docs/decisions/records/ci/required-job-step-execution-markers.md index 106267460..12572ed02 100644 --- a/docs/decisions/records/ci/required-job-step-execution-markers.md +++ b/docs/decisions/records/ci/required-job-step-execution-markers.md @@ -5,9 +5,9 @@ status: active since: '2026-08-10' supersedes: none superseded-by: none -rule: 'A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`''s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark ` as its FIRST act, and the job''s LAST step calls `ci-step-ran.sh assert --always --gated `, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step''s `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships.' -signals: 'required check green but no work done, step never ran but job green, Build & test green in seconds, EF migration integrity green without replaying, missing Run Main step marker, Unable to interpolate expression format(, dropped step docker-build, ci-step-ran.sh, marker file, expression delimiter in a required job · paths: `.gitea/workflows/docker-build.yml`, `scripts/ci-step-ran.sh`, `scripts/tests/test_ci_dropped_step_guard.py` · issues: #756, #751, #684' -mechanics: '`scripts/ci-step-ran.sh` owns the marker path so it exists ONCE and the write and the read cannot diverge. It is keyed on `GITHUB_JOB`/`GITHUB_RUN_ID` — REQUIRED, refusing rather than falling back to a reusable name — plus `GITHUB_RUN_ATTEMPT`. All three REFUSE rather than falling back to a reusable name. The third was warn-and-default until its presence was measured: grepping a log for the variable NAME proves nothing, and inferring it from the absence of a stderr warning proves nothing either (stderr capture was itself unestablished), so `assert` was made to print `Marker identity: job=… run=… attempt=… (from the runner)` on STDOUT and the answer was read off run 1916 for both required jobs. That line is retained as standing evidence. Do NOT justify the keying with #751''s "RUNNER_TEMP is /tmp, not a private per-job dir": that was measured on a job with no `container:` and does not transfer — these jobs get a fresh container, which is the primary protection, and the keying is defence in depth. Held by `scripts/tests/test_ci_dropped_step_guard.py`: static (marker set derived from the workflow equals the guard''s expectations, bucket matches each step''s `if:`, guard is last / has no `if:` / is not advisory / has no delimiter) and behavioural (the guard''s real command line executed against markers written by the steps'' real marker lines, dropping each key in turn).' +rule: 'A step the runner declines to interpolate is DROPPED and the job still concludes `success` (`ci.workflow-run-body-no-expressions`). In `review-verdict.yml` that is fail-CLOSED — the required status is absent and the merge is blocked. In `docker-build.yml`''s `test` and `migrations` it is fail-OPEN: those are the other two required contexts on `main`, so the check reports green having done no work. So in those two jobs every `run:` step that is not `continue-on-error: true` calls `"$GITHUB_WORKSPACE/scripts/ci-step-ran.sh" mark ` as its FIRST act, and the job''s LAST step calls `ci-step-ran.sh assert --always --gated `, which fails the job when an expected key was never recorded. PER STEP, not per job: a marker written by the first step only proves the job started, while the drop that costs something is `Test` or the migration replay. The guard carries NO `if:` — the default `success()` is the wanted condition, because a genuine failure in an early step legitimately skips every later one and an `always()` guard would announce a false "these steps never executed" on every ordinary red build; the invariant that makes the omission safe is that the guard is skipped only when an earlier step FAILED, which already fails the job, so guard-skipped implies job-red and every path to a green job runs the guard. Separately and independently, no `${{` OPENER may appear in any `run:` body of those two jobs OR of `build` — the drop mechanism requires the opener, so banning it makes the class unreachable rather than merely caught, and an UNCLOSED opener triggers the same rewrite as a well-formed pair. Pass values in through the step''s `env:`, which is interpolated per value. The two halves have DIFFERENT scopes on purpose: markers cover the required pair, while the ban also covers `build`, whose `Smoke + IPTV E2E` step runs AFTER the image is pushed, so a drop there publishes a release candidate that was never booted and that `DeployStack jazz-media` then promotes. `functional-e2e` is delimiter-free but deliberately excluded (advisory by declaration), and `api-docs`/`format` keep one `github.base_ref` each and gate nothing that ships. The ban is enforced on the RELEASE PATH itself, not only in review (#767): a `scan` job runs the PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and no image is published. A guard STEP inside `build` was tried first and is wrong — a step cannot protect the job it publishes from, and "my body has no opener so I cannot be dropped" is circular when only the PR-only test enforces that. The pytest in `script-tests` remains, but it is `on: pull_request` and not a required context, so it alone left the tag path unchecked.' +signals: 'required check green but no work done, step never ran but job green, Build & test green in seconds, EF migration integrity green without replaying, missing Run Main step marker, Unable to interpolate expression format(, dropped step docker-build, ci-step-ran.sh, marker file, expression delimiter in a required job · paths: `.gitea/workflows/docker-build.yml`, `scripts/ci-step-ran.sh`, `scripts/tests/test_ci_dropped_step_guard.py`, `scripts/tests/test_ci_release_path_scan_job.py` · issues: #756, #751, #684, #767' +mechanics: '`scripts/ci-step-ran.sh` owns the marker path so it exists ONCE and the write and the read cannot diverge. It is keyed on `GITHUB_JOB`/`GITHUB_RUN_ID` — REQUIRED, refusing rather than falling back to a reusable name — plus `GITHUB_RUN_ATTEMPT`. All three REFUSE rather than falling back to a reusable name. The third was warn-and-default until its presence was measured: grepping a log for the variable NAME proves nothing, and inferring it from the absence of a stderr warning proves nothing either (stderr capture was itself unestablished), so `assert` was made to print `Marker identity: job=… run=… attempt=… (from the runner)` on STDOUT and the answer was read off run 1916 for both required jobs. That line is retained as standing evidence. Do NOT justify the keying with #751''s "RUNNER_TEMP is /tmp, not a private per-job dir": that was measured on a job with no `container:` and does not transfer — these jobs get a fresh container, which is the primary protection, and the keying is defence in depth. Held by `scripts/tests/test_ci_dropped_step_guard.py`: static (marker set derived from the workflow equals the guard''s expectations, bucket matches each step''s `if:`, guard is last / has no `if:` / is not advisory / has no delimiter) and behavioural (the guard''s real command line executed against markers written by the steps'' real marker lines, dropping each key in turn). The release-path `scan` job (#767) runs the existing PyYAML-based ban test rather than a second implementation, so there is no drift surface; `scripts/tests/test_ci_release_path_scan_job.py` holds the WIRING instead — that `build` needs it AND that `build`''s own `if:` carries no `always()`/`!cancelled()`/`failure()` (which would downgrade the `needs:` edge to mere ordering), that `scan` carries no job-level `if:` (one excluding the tag push restores the hole, one skipping the job skips `build` too) and is not advisory at STEP or JOB level, and that its own run bodies are delimiter-free. Its load-bearing test is an EXECUTION PROBE, not a shape assertion: it runs the scan step''s real `run:` body with the full env the runner would give it (workflow, job AND step `env:` tiers) against a copy of the repo whose `Smoke` body carries an injected delimiter, and requires a non-zero exit, with a clean-tree negative control. Shape assertions were tried and lost repeatedly — from `echo`ing the command to `PYTEST_ADDOPTS` one env tier up — so do NOT replace the probe with cheaper checks about the command. Two tiers cannot be reached from inside pytest at all and are handled differently: a step writing to `$GITHUB_ENV` is BANNED by test, and repo-root pytest configuration (`pytest.ini` `addopts`, `pytest_collection_modifyitems`) can deselect any test including the guards, so the positive control is a SHELL step — `scripts/ci-prove-ban-detects.sh` poisons the checked-out workflow in the REAL checkout, re-runs the ban test, and vouches ONLY for the ban test''s `build` parametrisation failing — any other outcome (exit 5 from a total deselect, exit 2 from a collection error, an unrelated test failing) is a REFUSAL, not a pass, because each weaker reading was a live bug in an earlier draft and the deselection disarm it exists to catch exits 5 rather than 1. That script is itself positively controlled — `test_the_PROOF_SCRIPT_itself_refuses_when_the_ban_is_deselected` and `test_the_PROOF_SCRIPT_refuses_when_the_WRONG_test_fails` cover the two refusal branches a disarm actually lands on, each verified by making that branch alone unreachable — since it was for a while the one guard exercised only on the happy path. The third branch (pytest passing outright) has no control and does not need one: neutering it falls through to the exit-code branch, which still refuses. A copy-based proof is not equivalent: it does not inherit the repo-root config a disarm would live in. Its steps carry markers and a trailing assert of their own, verified by the same drop-each-key-in-turn behavioural pattern. CARVE-OUT: the "fresh container is the primary protection, keying is defence in depth" reasoning above does NOT cover `scan` — it has no `container:` and runs on `small`, where RUNNER_TEMP is the shared host /tmp, so for that job the run-id/attempt keying is the ONLY protection. Residual: a single-job re-run that does not increment GITHUB_RUN_ATTEMPT would find the prior attempt''s marker file and the assert would pass even with the pytest step dropped. Identity on that lane was measured, not assumed — run 1929 printed `Marker identity: job=scan run=1929 attempt=1 (from the runner)`.' --- **Why per step, when #756 proposed per job.** A job-start marker answers "did this job begin", which @@ -75,12 +75,42 @@ ban. The claim not to repeat is the draft's dichotomy ("give up interpolation or remains uncovered is `api-docs` and `format`, whose one `github.base_ref` each sits in a detect step that gates nothing that ships, and `functional-e2e`, which is advisory by declaration. -**And the `build` ban is review-time, not fail-closed — do not read it as more.** It is enforced by -`script-tests`, which is `on: pull_request` and is NOT a required context, so nothing re-checks it -when a release is actually cut and `build` carries no markers or guard. A delimiter that reached -`main` would still drop `Smoke` on the tag build and report green. Closing that needs a third -bucket modelling `Smoke`'s publish-ref condition — deferred to ersatztv#767 rather than shipped -unmeasured onto the release path, since every other claim in this record is backed by a live probe. -Do NOT repeat the cost estimate an earlier draft gave for it ("would require pushing a real `v*` -tag"): `build` runs on every push to `main` and `Smoke`'s ref condition admits `refs/heads/main`, so -both directions are measurable without cutting a release. +**The `build` ban is now fail-closed on the release path (ersatztv#767 — this was the open +residual).** It used to be enforced only by `script-tests`, which is `on: pull_request` and is not a +required context, so nothing re-checked it when a release was actually cut: a delimiter that reached +`main` would still drop `Smoke` on the tag build and report green. A `scan` job now runs the +PyYAML-based ban test and `build` lists it in `needs:`, so a delimiter means `build` never runs and +no image is published. + +**Two designs were tried, and the first one's failures are the reusable part.** The first put a +bespoke stdlib scanner in `build` itself as an unconditional step before `Build and push`. Two +independent reviews rejected it on two counts, both easy to re-invent: + +- **A guard step cannot protect the job it lives in.** `build` publishes, so a guard step there is + fail-OPEN if the runner drops it. The defence offered — "the guard's own body has no opener, so it + cannot be dropped" — is circular, because the only thing enforcing that property was the same + PR-only, non-required test being backstopped. A `needs:` edge is not circular: a red job skips its + dependents by construction. +- **A hand-written parser was strictly weaker than the check it backstopped.** It hand-parsed YAML to + avoid provisioning PyYAML on `build`'s bare runner, and review found ~10 false NEGATIVES in one + round (flow mappings, a quoted `"run":` key, aliases, multiline quoted scalars). For a security + gate only false negatives matter, so this was worse than useless — it looked like enforcement. Do + not re-attempt a bespoke scanner to save provisioning a dependency; run the real test. + +**Why this needs no third marker bucket.** The deferral assumed the answer had to be markers on +`build`, requiring a bucket that models `Smoke`'s publish-ref `if:`. It does not: the delimiter class +is a *static* property of the workflow text, so a job that reads the text catches it without +modelling any `if:`. The marker buckets are unchanged. Per-step markers on `build` remain a genuine +smaller residual — they would catch a drop caused by something other than a delimiter. + +**What this does not claim.** That no step can ever fail to run for another reason. The `scan` job's +own steps carry markers and a trailing assert, which moves the terminal assumption rather than +removing it: to fail open you must now drop the pytest step AND the assert step, not either alone. + +**Measured, not assumed** — runs **1928** (poisoned) and **1929** (control), `workflow_dispatch` on +scratch branches, 2026-08-11. The arrangement: a `workflow_dispatch` on a scratch branch whose `Smoke` body +carries a deliberate delimiter must redden `scan` and leave `build` skipped, and the same dispatch +without the poison must pass. Note that "no image was published" is NOT part of the evidence — on a +scratch ref `Build and push` has `push: false` regardless, so that conjunct could not have come out +the other way; the discriminating observation is `scan` red and `build` skipped. Do NOT repeat the +cost estimate an earlier draft gave ("would require pushing a real `v*` tag"). diff --git a/docs/decisions/records/ci/small-lane-git-only.md b/docs/decisions/records/ci/small-lane-git-only.md index f7de79308..f1a5cc3f7 100644 --- a/docs/decisions/records/ci/small-lane-git-only.md +++ b/docs/decisions/records/ci/small-lane-git-only.md @@ -32,5 +32,5 @@ mechanics: sum-of-caps rule (#406/#604); second jazz runner at `--cpu-shares=128 works only because it lands after load clears, so a capacity problem read as a bug in the guard. A job that fails with zero log output is evidence about the runner, not about the job. - **#574's skip-task queueing does not return** by moving `build` back to `ubuntu-latest`: - `needs: [test, migrations]` means it cannot be dispatched until the jobs it would have queued behind + `needs: [test, migrations, scan]` means it cannot be dispatched until the jobs it would have queued behind have already finished. diff --git a/scripts/ci-prove-ban-detects.sh b/scripts/ci-prove-ban-detects.sh new file mode 100755 index 000000000..57b557028 --- /dev/null +++ b/scripts/ci-prove-ban-detects.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Prove — at runtime, every run — that the delimiter ban actually DETECTS a delimiter (ersatztv#767). +# +# WHY THIS EXISTS AS A SHELL STEP RATHER THAN A TEST. The `scan` job's guarantee used to be a pytest +# that asserted the ban command works. Review then disarmed the whole gate with ONE new file at the +# repo root — a `pytest.ini` carrying `addopts = -k "not delimiter_banned"`, or a `conftest.py` with +# `pytest_collection_modifyitems` — which deselects the ban test AND the tests guarding it, leaving +# every job green while a delimiter sits in `build`'s `Smoke` body and the image publishes. That tier +# sits above every `env:` a test can reconstruct, and above the tests themselves: anything living +# inside pytest can be deselected by pytest's own configuration. So the positive control lives here. +# +# IT RUNS IN THE REAL CHECKOUT, NOT A COPY, and that is the whole point. An isolated copy does not +# inherit the repo-root `pytest.ini`/`conftest.py`/`pyproject.toml` that a disarm would live in, so a +# copy-based proof reports the ban healthy while the job's actual invocation is deselected — measured, +# and it is why the first version of this script was wrong. Same cwd, same rootdir, same config and +# the same BAN INVOCATION as the run being vouched for — not the same command line, since the +# workflow step also runs the wiring tests and this runs only the ban file. Only the workflow file +# differs, and only for a moment. +# +# The workflow file is restored by an EXIT trap. That covers a normal exit and SIGTERM (measured: +# `git status` clean afterwards); it does NOT cover SIGKILL — a cancelled or OOM-killed job leaves +# the checkout poisoned. Do not upgrade this to "cannot leave a poisoned tree behind": it can. +# It stays harmless for a reason worth stating rather than assuming. The workspace that could be +# reused is THIS job's own, on the persistent `small` lane (`scan` has no `container:` — see the +# carve-out in `ci.required-job-step-execution-markers`), not `build`'s, which checks out fresh in a +# separate job. A leftover poisoned workflow there makes the NEXT `scan` red rather than quiet, and +# `actions/checkout` restores it anyway. Fail-closed in both directions. +# +# The failure direction is the safe one throughout: if this script cannot do its work (no python3, a +# moved anchor, an unwritable tree) it exits non-zero and the release is blocked, never silently +# skipped — `set -euo pipefail` extends that to the unexpected cases. +# +# AND IT ONLY VOUCHES FOR THE EXACT EXPECTED FAILURE. Three outcomes are distinguished below: pytest +# passing (the ban is not enforcing), pytest failing for a NON-test reason (collection error, nothing +# collected — cannot prove anything), and pytest failing on some OTHER test while the ban test itself +# was deselected. Only the ban test's `[build]` parametrisation failing counts as proof. Both weaker +# readings were live bugs in earlier drafts of this file: "non-zero means it noticed" reported OK on a +# collection error, and a later draft still fell through to OK when an unrelated test reddened. +# +# NOTE ON THE OPENER: this file must never contain the literal two-character expression opener, both +# because `scan`'s own run bodies are asserted delimiter-free and because a file containing it is the +# very hazard under test. It is CONSTRUCTED from its parts below. +set -euo pipefail + +BAN_TEST="scripts/tests/test_ci_dropped_step_guard.py" +WORKFLOW=".gitea/workflows/docker-build.yml" +# The line the poison is injected above, inside `build`'s `Smoke + IPTV E2E` body. Kept in one place +# so that rewriting that step fails loudly here instead of silently poisoning nothing. +ANCHOR='IMG="${IMAGE}:${SMOKE_SHORT_SHA}"' + +cd "${GITHUB_WORKSPACE:-$(pwd)}" + +backup="$(mktemp)" +cp "$WORKFLOW" "$backup" +restore() { cp "$backup" "$WORKFLOW"; rm -f "$backup"; } +trap restore EXIT + +python3 - "$WORKFLOW" "$ANCHOR" <<'PY' +import sys +from pathlib import Path + +path, anchor = Path(sys.argv[1]), sys.argv[2] +text = path.read_text() +if anchor not in text: + sys.exit( + "::error::ci-prove-ban-detects.sh could not find its anchor in the workflow. The Smoke step " + "was rewritten without updating this script, so the proof would poison nothing and pass " + "vacuously. Refusing." + ) +# Built from parts on purpose: the literal opener must not appear in this repo's own scripts. +opener = chr(36) + "{" + "{" +path.write_text(text.replace(anchor, f"# {opener} steps.meta.outputs.short }}}}\n {anchor}", 1)) +PY + +echo "Proving the delimiter ban detects an injected opener (ersatztv#767)…" +output="$(mktemp)" +trap 'restore; rm -f "$output"' EXIT +set +e +PYTHONPATH=. python3 -m pytest "$BAN_TEST" -q >"$output" 2>&1 +detected=$? +set -e + +# THREE OUTCOMES, EXHAUSTIVELY, and only the last one is a pass. "Non-zero means it noticed" is not +# good enough — that was this script's own first bug, and an earlier draft of this very block still +# fell through to OK when pytest exited 1 for a DIFFERENT test's failure while the ban test was +# deselected. Requiring the exact expected failure is what makes this a proof rather than a liveness +# check. (`[build]` because the ban is parametrised per job, and `build` is the one whose dropped +# `Smoke` publishes an unsmoked image.) +EXPECTED='test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body\[build\]' + +if [ "$detected" -eq 0 ]; then + echo "::error::THE DELIMITER BAN IS NOT ENFORCING. ${BAN_TEST} PASSED on a tree whose build/Smoke body contains an expression delimiter, so the release-path gate would let an unsmoked candidate publish. Causes seen before: a repo-root pytest.ini/conftest.py/pyproject.toml deselecting it, a neutered opener regex, 'build' dropped from DELIMITER_BAN_JOBS, or the test renamed. This check runs in the real checkout and outside pytest on purpose — anything inside pytest can be deselected by pytest's own configuration." >&2 + sed 's/^/ /' "$output" >&2 + exit 1 +elif [ "$detected" -ne 1 ]; then + echo "::error::ci-prove-ban-detects.sh could not prove anything: pytest exited ${detected} on the poisoned tree, which is not a test failure (2=collection error, 3=internal, 4=usage, 5=nothing collected). The ban may well be fine, but this check cannot vouch for it, so it refuses rather than reporting a pass it did not earn." >&2 + sed 's/^/ /' "$output" >&2 + exit 1 +elif ! grep -qE "$EXPECTED" "$output"; then + echo "::error::ci-prove-ban-detects.sh saw a test failure on the poisoned tree, but NOT the expected one. The ban test's build parametrisation did not fail, so something else did — the ban itself may be deselected or renamed while an unrelated test reddens. Refusing to read that as proof." >&2 + sed 's/^/ /' "$output" >&2 + exit 1 +fi +echo "OK: the ban detected the injected delimiter in build/Smoke (pytest exit 1, build parametrisation failed) — the gate is enforcing." diff --git a/scripts/ci-step-ran.sh b/scripts/ci-step-ran.sh index c9d3c8a63..491ec8351 100755 --- a/scripts/ci-step-ran.sh +++ b/scripts/ci-step-ran.sh @@ -96,9 +96,17 @@ EOF # and the same for `migrations`. Both required jobs, on the lane that matters. # # That measurement is what promoted it from warn-and-default to REQUIRED, which is why the residual -# this comment used to describe — a rerun inheriting attempt 1's markers — no longer exists. If a -# future runner stops exporting any of the three, every job reddens with a message naming the -# variable; that is loud, instantly diagnosable, and the correct direction for a required check. +# this comment used to describe — a rerun inheriting attempt 1's markers — no longer exists FOR THE +# CONTAINER JOBS. If a future runner stops exporting any of the three, every job reddens with a +# message naming the variable; that is loud, instantly diagnosable, and the correct direction for a +# required check. +# +# EXCEPT ON A LANE WITH NO `container:` (ersatztv#767). The `scan` job runs on `small` with no +# container, so RUNNER_TEMP is the shared host /tmp and the keying below is the ONLY thing separating +# runs, not defence in depth on top of a fresh filesystem. There the rerun residual DOES still exist: +# a single-job rerun that does not increment GITHUB_RUN_ATTEMPT would find the previous attempt's +# marker file. See the carve-out in `ci.required-job-step-execution-markers`; do not read the +# paragraph above as covering that job. marker_path() { local missing="" [ -n "${GITHUB_JOB:-}" ] || missing="$missing GITHUB_JOB" diff --git a/scripts/tests/test_ci_release_path_scan_job.py b/scripts/tests/test_ci_release_path_scan_job.py new file mode 100644 index 000000000..ebd3be911 --- /dev/null +++ b/scripts/tests/test_ci_release_path_scan_job.py @@ -0,0 +1,638 @@ +r"""The `scan` job — the delimiter ban made fail-CLOSED on the release path (ersatztv#767). + +WHAT THIS IS PROTECTING. #756 brought `build` into the delimiter ban, because a dropped +`Smoke + IPTV E2E` publishes a release candidate that was never booted and reports the job green. +But the ban was enforced ONLY by `test_the_delimiter_banned_jobs_have_NO_expression_delimiter_in_any_run_body` +in `script-tests` — `on: pull_request`, not a required context. Nothing re-checked it on a `v*` tag +push, which is exactly when the candidate is published. + +WHY A JOB AND NOT A STEP IN `build`, and why this file is structural. The first cut of #767 put a +bespoke stdlib scanner in `build` itself. Two independent reviews killed it on two counts, and both +are worth keeping written down because both are easy to re-invent: + + * A guard step inside `build` cannot protect `build`. If the runner drops it, the job carries on + and publishes — fail-OPEN. The defence offered was "the guard's own body has no opener, so it + cannot be dropped", but the only thing enforcing THAT was the same PR-only test being + backstopped. Circular. As a `needs:` of `build`, a red here means `build` never runs at all. + * The bespoke scanner hand-parsed YAML (to avoid provisioning PyYAML on `build`'s bare runner) and + had ~10 false NEGATIVES within one review round — flow mappings, a quoted `"run":` key, aliases, + multiline quoted scalars. It was strictly WEAKER than the check it backstopped, in the only + direction that matters. The fix was to delete it and run the real PyYAML-based test, which needs + no second definition of "what is a `run:` body" and so has no drift surface. + +The detection ALGORITHM is not reimplemented here — it lives in `test_ci_dropped_step_guard.py`, and +this job runs that file. What this file holds is the WIRING that makes the ban fail-closed (the job +exists, `build` depends on it, nothing can skip it, its own steps cannot be silently dropped) plus +ONE end-to-end probe that executes the scan step's real command against a poisoned copy of the repo +and requires it to fail. + +THAT PROBE IS THE LOAD-BEARING TEST, and it exists because shape assertions lost twice. Round 2 +replaced a substring check with checks ABOUT the command — bare-token argument, no `-k`, no `||` — +and round 3 then disarmed the gate seven more ways that all left the suite green: `echo`ing the +command instead of running it, flags moved past a `\` continuation, a trailing `exit 0`, +`if false; then … fi`, `set +e`, and `PYTEST_ADDOPTS` in the step's `env:`. Two further disarms lived +in the sibling module where no shape check could ever reach — the ban test's parametrize list swapped +to `MARKED_JOBS`, and its opener regex neutered. Running the command settles every one of them, +because it asks the only question that matters: with a delimiter in `build`'s `Smoke` body, does this +command fail? The shape checks are kept as a faster, more specific signal, not as the guarantee. + +ROUND 4 THEN FOUND THE TWIN OF THE ROUND-3 FIX, which is the reusable lesson here: fixing the STEP +`env:` tier did not generalise, and the same `PYTEST_ADDOPTS` disarm placed one tier up — on the JOB +— defeated the probe, because the probe reconstructed only the step's env. It now layers all three +tiers (workflow, job, step). The same round found that `needs:` is not by itself a gate: an +`always()` in `build`'s `if:` downgrades the edge to mere ordering, and the delimiter ban does not +cover `if:` expressions, so nothing else would have objected. + +ENUMERATING THAT LIST THEN FOUND A FOURTH TIER the probe cannot ever reach: a step writing to +`$GITHUB_ENV` injects into LATER steps at runtime, so it is invisible to any static reconstruction of +the workflow text. Measured — `PYTEST_ADDOPTS` supplied that way makes the ban command exit 0 on a +poisoned tree while the probe reports healthy. That one is BANNED rather than modelled +(`test_no_step_in_the_scan_job_writes_to_GITHUB_ENV`), because emulating the runner's semantics would +be a second implementation of precisely the kind #767 already deleted once. The rule after any fix +here: enumerate the tiers and the twin, and where a tier cannot be observed, forbid it. + +(Two disarms review reported were checked and are NOT real: `--ignore=` and a `conftest.py` +`collect_ignore` do not suppress a file pytest was given explicitly as an argument — measured, the +ban test still ran and still failed. Recorded so they are not re-litigated. But note what refuting +them did NOT establish: the WORKING attacks through that same configuration channel — a repo-root +`pytest.ini` `addopts`, or `pytest_collection_modifyitems` — were simply never tried, and both +disarm everything here. Refuting two variants of a channel is not clearing the channel.) + +WHAT THIS DOES NOT CLAIM. That no step can ever fail to run for a reason other than the interpolation +drop. This job's own steps carry #756 markers and a trailing assert, so the regress terminates where +the sibling guards' does — to fail open you must drop the pytest step AND the assert step. And the +probe runs the command, not the RUNNER: that a red `scan` actually skips `build` is a live +measurement recorded on the issue, which no test here can establish. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import uuid +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / ".gitea" / "workflows" / "docker-build.yml" +SCRIPT = REPO_ROOT / "scripts" / "ci-step-ran.sh" + +_DOC = yaml.safe_load(WORKFLOW.read_text()) +_OPENER = re.compile(r"\$\{\{") +_MARK = re.compile(r'ci-step-ran\.sh"?\s+mark\s+(\S+)') + +JOB = "scan" +BAN_TEST_FILE = "scripts/tests/test_ci_dropped_step_guard.py" + + +def _job(): + assert JOB in _DOC["jobs"], f"the `{JOB}` job is gone — the release path is unguarded again" + return _DOC["jobs"][JOB] + + +def _steps(): + return _job()["steps"] + + +def _run_steps(): + return [s for s in _steps() if s.get("run")] + + +def _guard(): + """The trailing assert step, located by CONTENT — never by index, so that + `test_the_guard_is_the_LAST_step` is not true by construction.""" + hits = [s for s in _run_steps() if "ci-step-ran.sh assert" in s["run"]] + assert len(hits) == 1, f"expected exactly 1 assert step in `{JOB}`, found {len(hits)}" + return hits[0] + + +def _marked(): + out = [] + for s in _run_steps(): + m = _MARK.search(s["run"]) + if m: + out.append((s, m.group(1))) + return out + + +# ------------------------------------------------------------------------------------------------ +# WIRING — the properties that make the ban fail-closed +# ------------------------------------------------------------------------------------------------ + + +def test_build_DEPENDS_on_the_scan_job(): + """This single edge is the whole fail-closed property. + + Without it the scan is advisory: it could go red while `build` publishes anyway. + """ + needs = _DOC["jobs"]["build"]["needs"] + needs = [needs] if isinstance(needs, str) else needs + assert JOB in needs, f"`build` no longer needs `{JOB}` — a red scan would not stop a release" + + +def test_build_does_not_NEUTRALISE_the_edge_with_an_always_style_condition(): + """`needs:` alone is not the gate — `always()` downgrades it to mere ordering. + + A dependent whose `if:` is `always()` (or `!cancelled()`, or `failure()`) runs even when the job + it needs went red, so `build` would publish with a red `scan`. Reproduced by review: + `if: ${{ always() && github.event_name != 'pull_request' }}` left the whole suite green while the + edge no longer gated anything. The delimiter ban does not cover `if:` expressions, so nothing + else in this repo would have objected either. + """ + cond = str(_DOC["jobs"]["build"].get("if", "")) + for escape in ("always(", "!cancelled(", "cancelled(", "failure("): + assert escape not in cond.replace(" ", ""), ( + f"`build`'s `if:` contains {escape!r}, which makes it run despite a red `{JOB}` — the " + f"`needs:` edge stops being a gate: {cond!r}" + ) + + +def test_the_scan_job_runs_the_out_of_pytest_positive_control(): + """The proof that the ban would DETECT must be invoked, and must not be a test. + + Everything else in this file lives inside pytest, and pytest's own configuration outranks it: a + repo-root `pytest.ini` with `addopts = -k "not delimiter_banned"`, or a `conftest.py` with + `pytest_collection_modifyitems`, deselects the ban test AND every test guarding it, leaving the + suite green with a delimiter in `Smoke`. Measured, both forms. `scripts/ci-prove-ban-detects.sh` + is the answer precisely because it is a shell step: it cannot be deselected. + + This test can itself be deselected, of course — which is why it is not what makes the gate safe. + It is here so that REMOVING the step from the workflow is loud in review; the step's own presence + in the job, plus its `selfcheck` marker in the trailing assert, is what makes it loud at runtime. + """ + script = "scripts/ci-prove-ban-detects.sh" + assert (REPO_ROOT / script).is_file(), f"{script} is missing" + assert os.access(REPO_ROOT / script, os.X_OK), f"{script} is not executable" + assert any(script in s["run"] for s in _run_steps()), ( + f"no step in `{JOB}` runs {script} — the ban is asserted to HOLD but never proved to DETECT" + ) + assert "selfcheck" in {k for _, k in _marked()}, ( + "the positive-control step records no marker, so a dropped one would go unnoticed" + ) + + +def test_no_step_in_the_scan_job_writes_to_GITHUB_ENV(): + """The one env tier the probe structurally CANNOT reconstruct, so it is banned instead. + + `_scan_body_and_env` layers the workflow, job and step `env:` blocks — all of which are visible + in the workflow text. A step writing `FOO=bar >> $GITHUB_ENV` injects into LATER steps at + runtime, which no static reconstruction can see. Measured: with `PYTEST_ADDOPTS` supplied that + way, the ban command exits 0 on a tree whose `Smoke` body is poisoned — a full disarm that the + execution probe reports as healthy, because the probe's subprocess never receives it. + + Banning the mechanism is the fix rather than modelling it: nothing in this job needs to pass a + value to a later step, and a guard that has to emulate the runner's `GITHUB_ENV` semantics would + be a second implementation of exactly the kind that #767 already deleted once. + """ + offenders = [s.get("name") for s in _run_steps() if "GITHUB_ENV" in s["run"]] + assert not offenders, ( + f"step(s) {offenders} in `{JOB}` write to GITHUB_ENV, which injects environment into later " + "steps at runtime where no static check — including this file's execution probe — can see " + "it. `PYTEST_ADDOPTS` supplied that way disarms the ban entirely." + ) + + +def test_the_scan_job_is_not_advisory_at_JOB_level(): + """`continue-on-error` on the JOB reports success to dependents even when it failed. + + The step-level check elsewhere in this file does not see this tier — the same one-tier-up miss + that let a job-level `env:` disarm the probe. + """ + assert not _job().get("continue-on-error"), ( + f"`{JOB}` is job-level advisory — a failure would report success to `build`" + ) + + +def test_the_scan_job_has_NO_job_level_if(): + """Two failure modes at once, in opposite directions. + + An `if:` that excludes the tag push would leave the release path unguarded — the exact hole + #767 closed. An `if:` that skipped it for any other reason would SKIP `build` too (a skipped + dependency skips its dependents), breaking every release. Neither is wanted: it always runs. + """ + job = _job() + assert "if" not in job, f"`{JOB}` must carry no job-level `if:`, found {job.get('if')!r}" + + +def _pytest_line() -> str: + """The ONE non-comment line that actually invokes pytest, from the job's real body. + + A substring test over the whole body is not enough, and that is not hypothetical: review + disarmed the gate three ways that all left the suite green — commenting the invocation out and + echoing instead (the filename still appears, in the comment), appending `-k 'not + delimiter_banned'`, and appending `|| true`. Each left `scan` green with the ban unchecked and + `build` publishing. So this locates the executable line and the caller asserts its shape. + """ + lines = [ + ln.strip() + for s in _run_steps() + for ln in s["run"].splitlines() + if "python3 -m pytest" in ln and not ln.strip().startswith("#") + ] + assert len(lines) == 1, f"expected exactly 1 pytest invocation in `{JOB}`, found {len(lines)}" + return lines[0] + + +def test_the_scan_job_actually_invokes_the_ban_test(): + """Otherwise the job is an expensive no-op that reports green. + + The path must appear as a BARE TOKEN on the pytest line — not merely somewhere in the body — + so that commenting the invocation out is a red. Renaming the ban test without updating the + workflow is a red here too, rather than a silently unguarded release path. + """ + assert (REPO_ROOT / BAN_TEST_FILE).is_file() + assert BAN_TEST_FILE in _pytest_line().split(), ( + f"`{JOB}` does not pass {BAN_TEST_FILE} to pytest as an argument: {_pytest_line()!r}" + ) + + +def _scan_body_and_env(): + """The ban step's REAL `run:` body, and the FULL env the runner would give it. + + ALL THREE TIERS, lowest precedence first: workflow `env:`, job `env:`, step `env:`. Reconstructing + only the step tier is not a smaller version of this — it is a hole, and review reproduced it: a + `PYTEST_ADDOPTS: -k "not delimiter_banned"` placed on the JOB disarmed the real gate (exit 0 with + a poisoned `Smoke` body) while every test here stayed green, because the probe never saw that + tier. It is the exact twin of the step-level `PYTEST_ADDOPTS` disarm caught one round earlier — + which is the lesson: after fixing one tier, enumerate the others rather than assuming the fix + generalised. + """ + step = next(s for s in _run_steps() if "python3 -m pytest" in s["run"]) + env = {} + for tier in (_DOC.get("env"), _job().get("env"), step.get("env")): + env.update({str(k): str(v) for k, v in (tier or {}).items()}) + return step["run"], env + + +def _repo_copy(tmp_path: Path) -> Path: + """A minimal executable copy of the repo: the workflow plus all of scripts/.""" + dst = tmp_path / "repo" + (dst / ".gitea" / "workflows").mkdir(parents=True) + for wf in (REPO_ROOT / ".gitea" / "workflows").glob("*.yml"): + shutil.copy2(wf, dst / ".gitea" / "workflows" / wf.name) + shutil.copytree(REPO_ROOT / "scripts", dst / "scripts") + return dst + + +# THE RECURSION FENCE. The scan step's real command runs this very file, whose two probe tests would +# each spawn the command again — unbounded recursion, discovered by running it. The fence lets the +# nested run skip just those two probes while still executing the ban test, which is the part that +# matters. Deliberately NOT a `-k`/`--ignore` on the nested command line: narrowing the command would +# mean the thing under test is no longer the command the workflow runs. +# +# IT IS A NONCE, NOT A CONSTANT. A constant fence is a one-line silent kill switch: exporting +# `ETV_SCAN_PROBE_NESTED=1` anywhere — a job `env:`, a runner default — would delete both +# load-bearing probes everywhere, with nothing going red, after which the disarms they exist to catch +# become unobservable again. So the parent writes a fresh value into the copied tree and passes the +# same value in the environment; the skip fires only when the two agree, which cannot happen outside +# a run this file itself spawned. +_FENCE = "ETV_SCAN_PROBE_NESTED" +_NONCE_FILE = ".scan-probe-nonce" + + +def _fence_active() -> bool: + value = os.environ.get(_FENCE) + if not value: + return False + try: + return (Path.cwd() / _NONCE_FILE).read_text().strip() == value + except OSError: + return False + + +_nested = pytest.mark.skipif( + _fence_active(), + reason="nested invocation of the scan step's own command — the outer run owns this probe", +) + + +def _run_the_real_scan_body(repo: Path, tmp_path: Path): + body, step_env = _scan_body_and_env() + env = { + "PATH": os.environ["PATH"], + "HOME": os.environ.get("HOME", str(tmp_path)), + "GITHUB_WORKSPACE": str(repo), + "RUNNER_TEMP": str(tmp_path / "runner"), + "GITHUB_JOB": JOB, + "GITHUB_RUN_ID": "990099", + "GITHUB_RUN_ATTEMPT": "1", + } + env.update(step_env) + # The fence is applied AFTER the workflow's own env tiers, so the file under test cannot control + # it in either direction — it could otherwise turn the nested probes back on (recursion) or off. + nonce = uuid.uuid4().hex + (repo / _NONCE_FILE).write_text(nonce) + env[_FENCE] = nonce + (tmp_path / "runner").mkdir(exist_ok=True) + return subprocess.run(["bash", "-c", body], cwd=repo, env=env, + capture_output=True, text=True) + + +@_nested +def test_the_scan_step_REALLY_FAILS_on_a_poisoned_workflow(tmp_path): + """THE POSITIVE CONTROL, and the one test here that is not a shape assertion. + + It executes the ban step's actual `run:` body — with its actual `env:` — against a copy of the + repo whose `build`/`Smoke` body carries an injected delimiter, and requires a non-zero exit. + + This exists because shape checks lost. An earlier round asserted things ABOUT the command + (the ban test's path is a bare token, no `-k`/`--deselect`, no `||`) and review then disarmed the + gate anyway, every time leaving the suite green: `echo`ing the command instead of running it, + flags moved past a `\\` continuation, a trailing `exit 0`, `if false; then … fi`, `set +e`, and + `PYTEST_ADDOPTS` in the step's `env:`. Enumerating shapes of a command cannot win; running the + command settles them together, including the two in the sibling module that no shape check could + reach (a parametrize list swapped to `MARKED_JOBS`, and a neutered opener regex). No count is + given on purpose — an earlier draft's tally disagreed with its own list, and the number was never + the point. + + The negative control below is what stops this passing for the wrong reason. And note the limit of + this probe: it runs the command in a COPY, so it does not inherit a repo-root pytest config — + that tier is covered by `scripts/ci-prove-ban-detects.sh`, which runs in the real checkout. + """ + repo = _repo_copy(tmp_path) + wf = repo / ".gitea" / "workflows" / "docker-build.yml" + text = wf.read_text() + anchor = ' IMG="${IMAGE}:${SMOKE_SHORT_SHA}"' + assert anchor in text, "anchor for the poison is gone — rewrite this control" + wf.write_text(text.replace(anchor, ' # ${{ steps.meta.outputs.short }}\n' + anchor, 1)) + + res = _run_the_real_scan_body(repo, tmp_path) + assert res.returncode != 0, ( + "the scan step PASSED on a workflow whose `build`/`Smoke` body contains an expression " + f"delimiter — the release-path gate is disarmed.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}" + ) + + +@_nested +def test_the_scan_step_PASSES_on_the_clean_workflow(tmp_path): + """The negative control for the test above. + + Without it, a scan step that failed for ANY reason — a missing interpreter, a bad path, a + syntax error in the body — would satisfy the positive control while proving nothing about + detection. This also demonstrates the copied tree is a working harness rather than a broken one. + """ + res = _run_the_real_scan_body(_repo_copy(tmp_path), tmp_path) + assert res.returncode == 0, ( + f"the scan step failed on a CLEAN workflow.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}" + ) + + +@_nested +def test_the_PROOF_SCRIPT_itself_refuses_when_the_ban_is_deselected(tmp_path): + """A positive control for the positive control — the one guard that had none. + + Everything else here is guarded by something; `ci-prove-ban-detects.sh` was guarded only by a + presence-and-executable-bit check, so a plausible "simplify" edit (relaxing the exit-code test + back to "any non-zero means it noticed") would silently reinstate a full disarm with every test + green. That is the "a guard only exercised on the happy path proves nothing" failure this file + argues about every other guard. + + The disarm reproduced here is the real one: a repo-root `conftest.py` that deselects the ban test + entirely, which makes pytest exit 5 (nothing collected) rather than fail — the exact reading that + made an earlier draft of the script report the gate healthy while an unsmoked candidate would + publish. + """ + repo = _repo_copy(tmp_path) + (repo / "conftest.py").write_text( + "def pytest_collection_modifyitems(config, items):\n" + " items[:] = [i for i in items if 'test_ci_dropped_step_guard' not in str(i.fspath)]\n" + ) + res = subprocess.run( + ["bash", str(repo / "scripts" / "ci-prove-ban-detects.sh")], + cwd=repo, env={**os.environ, "GITHUB_WORKSPACE": str(repo)}, + capture_output=True, text=True, + ) + assert res.returncode != 0, ( + "ci-prove-ban-detects.sh vouched for the gate while the ban test was deselected at the " + f"repo-root config tier.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}" + ) + # NON-ZERO IS NOT ENOUGH — the script could exit non-zero because its own harness broke (a moved + # anchor, an unwritable tree, no python3), which would make this test pass while proving nothing. + # Require one of its real verdicts. Measured, this path yields "pytest exited 5" (nothing + # collected), which is precisely the deselection disarm and precisely the reading an earlier draft + # mistook for success. + combined = res.stdout + res.stderr + # THE SPECIFIC branch, not a disjunction over the script's verdicts. Measured: a total deselect + # makes pytest exit 5, so this lands on "cannot prove anything". Accepting any verdict would let + # the scenario drift onto a branch it was not written for while still looking green — this file's + # own subject, one level down. + assert "could not prove anything" in combined, ( + "expected the cannot-prove branch (a total deselect makes pytest exit 5); got a different " + f"verdict, so this test no longer covers what it was written for.\nstdout:\n{res.stdout}\n" + f"stderr:\n{res.stderr}" + ) + + +@_nested +def test_the_PROOF_SCRIPT_refuses_when_the_WRONG_test_fails(tmp_path): + """The third branch, which the aggregate 'deselect ⇒ non-zero' control does not reach. + + Deselecting only the `[build]` parametrisation while some unrelated test fails gives pytest exit + 1 — a real test failure, just not the one that proves anything. An earlier draft read that as + success. Reproduced here because that branch was added to fix a live bug and was otherwise + exercised by nothing: making it unreachable left both guard files green. + """ + repo = _repo_copy(tmp_path) + (repo / "conftest.py").write_text( + "def pytest_collection_modifyitems(config, items):\n" + " items[:] = [i for i in items if 'in_any_run_body[build]' not in i.name]\n" + ) + # The unrelated failure has to live INSIDE the ban file: the script runs that file and nothing + # else, so a failing test in a sibling module is never collected and the run would exit 0 — + # landing on the "not enforcing" branch instead of the one under test. (First draft of this test + # did exactly that and was red for the wrong reason.) + ban = repo / BAN_TEST_FILE + ban.write_text( + ban.read_text() + + "\n\ndef test_an_unrelated_failure_for_this_probe():\n assert False\n" + ) + res = subprocess.run( + ["bash", str(repo / "scripts" / "ci-prove-ban-detects.sh")], + cwd=repo, env={**os.environ, "GITHUB_WORKSPACE": str(repo)}, + capture_output=True, text=True, + ) + combined = res.stdout + res.stderr + assert res.returncode != 0, ( + f"the script read an unrelated test's failure as proof.\n{combined}" + ) + # THE SPECIFIC branch. This scenario is built to land on "wrong test failed" (exit 1, no `[build]` + # failure); accepting "could not prove anything" too would let it drift onto the exit-5 branch and + # silently cover a branch it was not written for, while still looking green. + assert "NOT the expected one" in combined, ( + "expected the wrong-test-failed branch; got a different verdict, so this scenario no longer " + f"covers the branch it was written for.\n{combined}" + ) + + +@_nested +def test_the_PROOF_SCRIPT_passes_on_a_clean_tree(tmp_path): + """Negative control for the test above: it must not simply always fail.""" + repo = _repo_copy(tmp_path) + res = subprocess.run( + ["bash", str(repo / "scripts" / "ci-prove-ban-detects.sh")], + cwd=repo, env={**os.environ, "GITHUB_WORKSPACE": str(repo)}, + capture_output=True, text=True, + ) + assert res.returncode == 0, ( + f"ci-prove-ban-detects.sh failed on a clean tree.\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}" + ) + assert (repo / ".gitea" / "workflows" / "docker-build.yml").read_text() == ( + WORKFLOW.read_text() + ), "the script did not restore the workflow file it poisoned" + + +def test_the_pytest_invocation_cannot_DESELECT_or_swallow_its_result(): + """Selection flags and shell escapes are the cheap ways to keep the job green while it checks + nothing. `-k`/`-m`/`--deselect`/`--ignore` can drop the ban test from a run that still reports + passes; `|| true` and friends discard the exit status the `needs:` edge depends on.""" + line = _pytest_line() + tokens = line.split() + # Only the tokens AFTER `pytest` are pytest's own arguments. Checking the whole line would flag + # the `-m` in `python3 -m pytest`, which is how the interpreter is invoked — a false positive + # that would make this test red on the correct command. + args = tokens[tokens.index("pytest") + 1:] + banned = {"-k", "-m", "--deselect", "--ignore", "--collect-only", "--co"} + assert not (banned & set(args)), f"pytest invocation may deselect tests: {line!r}" + for op in ("||", "&&", ";", "|"): + assert op not in tokens, f"pytest exit status is not decisive — {op!r} in {line!r}" + + +def test_the_ban_SCOPE_still_covers_build(): + """The whole release-path property rests on one literal in the sibling module. + + `DELIMITER_BAN_JOBS` is what the ban test parametrises over. Drop `"build"` from it and the + suite still reports passes while nothing checks the job that publishes the image — green, and + the release path unguarded. Nothing else in scripts/tests referenced that constant, so this is + the pin. (An earlier design imported the tuple, which pinned it as a side effect; the import + went away with that design and took the protection with it.) + """ + import scripts.tests.test_ci_dropped_step_guard as ban + + assert "build" in ban.DELIMITER_BAN_JOBS, ( + "`build` was dropped from DELIMITER_BAN_JOBS — the `scan` job would go green while the job " + "that publishes the release candidate is unchecked (ersatztv#767)" + ) + + +def test_no_step_in_the_scan_job_is_advisory(): + """`continue-on-error: true` would make the whole gate a no-op while every other test here + stayed green — it is the cheapest way to accidentally disarm this.""" + offenders = [s.get("name") for s in _steps() if s.get("continue-on-error")] + assert not offenders, f"advisory step(s) in `{JOB}`: {offenders}" + + +@pytest.mark.parametrize( + "step_name", + [s.get("name", "?") for s in yaml.safe_load(WORKFLOW.read_text())["jobs"][JOB]["steps"] + if s.get("run")], +) +def test_every_run_body_in_the_scan_job_is_delimiter_free(step_name): + """The guard must not be vulnerable to the defect it guards against. + + Not a proof that it always runs — a construction argument about ONE mechanism, the same axiom + the sibling guards rest on. It is asserted per step so a failure names which step regressed. + """ + step = next(s for s in _run_steps() if s.get("name", "?") == step_name) + assert not _OPENER.search(step["run"]), ( + f"step {step_name!r} of `{JOB}` contains an expression delimiter; the runner would rewrite " + "the whole body and DROP the step while reporting success (ersatztv#751). Pass values " + "through `env:`, which is interpolated per value." + ) + + +# ------------------------------------------------------------------------------------------------ +# THE JOB'S OWN DROPPED-STEP GUARD +# ------------------------------------------------------------------------------------------------ + + +def test_every_consequential_step_marks_itself(): + """Every `run:` step except the guard records that it executed.""" + marked = {s.get("name") for s, _ in _marked()} + expected = {s.get("name") for s in _run_steps() if s is not _guard()} + assert marked == expected, f"unmarked step(s) in `{JOB}`: {expected - marked}" + + +def test_the_guard_expectations_match_the_markers_exactly(): + """The set the guard waits for IS the set the steps write — derived from the workflow, not + restated here, so adding a step without a marker is a red.""" + argv = _guard()["run"].split() + assert "--always" in argv, argv + always = argv[argv.index("--always") + 1:] + assert "--gated" not in argv, "every step in this job is unconditional; there is nothing to gate" + assert sorted(always) == sorted(k for _, k in _marked()) + + +def test_the_guard_is_the_LAST_step(): + assert _steps()[-1] is _guard(), "the assert must run after the steps it checks" + + +def test_the_guard_has_no_if(): + """Same reasoning as the sibling guards: the default `success()` is wanted, because a genuine + early failure legitimately skips later steps and already fails the job.""" + assert "if" not in _guard() + + +# ------------------------------------------------------------------------------------------------ +# BEHAVIOURAL — the guard's REAL command line, against the steps' REAL marker lines +# ------------------------------------------------------------------------------------------------ + + +def _mark_line(step) -> str: + """The step's own marker line, verbatim from the workflow — never rebuilt in Python, so a + drift between the workflow and the script cannot hide behind a test that composed its own.""" + return next(ln for ln in step["run"].splitlines() if _MARK.search(ln)).strip() + + +def _env(tmp_path, **extra): + env = { + "PATH": os.environ["PATH"], + "GITHUB_WORKSPACE": str(REPO_ROOT), + "RUNNER_TEMP": str(tmp_path), + "GITHUB_JOB": JOB, + "GITHUB_RUN_ID": "424242", + "GITHUB_RUN_ATTEMPT": "7", + } + env.update(extra) + return {k: v for k, v in env.items() if v is not None} + + +def _run(script: str, env): + return subprocess.run(["bash", "-c", script], cwd=REPO_ROOT, env=env, + capture_output=True, text=True) + + +def test_the_guard_PASSES_when_every_step_ran(tmp_path): + env = _env(tmp_path) + for step, _ in _marked(): + assert _run(_mark_line(step), env).returncode == 0 + res = _run(_guard()["run"], env) + assert res.returncode == 0, res.stderr + + +@pytest.mark.parametrize("dropped", [k for _, k in _marked()]) +def test_the_guard_FAILS_when_a_step_was_dropped(tmp_path, dropped): + """The positive control. Drop each key in turn — the guard must go red and NAME it. + + A guard only ever exercised on the happy path is indistinguishable from one that passes + unconditionally, which is the failure this whole mechanism exists to remove. + """ + env = _env(tmp_path) + for step, key in _marked(): + if key != dropped: + assert _run(_mark_line(step), env).returncode == 0 + res = _run(_guard()["run"], env) + assert res.returncode != 0, f"guard passed despite '{dropped}' never running: {res.stdout}" + # BOTH streams: the script's `::error::` lands on stdout here while other diagnostics go to + # stderr, and a test that picked the wrong one would assert on an empty string and pass for the + # wrong reason on any message change. + assert dropped in (res.stdout + res.stderr), (res.stdout, res.stderr) + + +def test_the_guard_REFUSES_to_pass_with_no_expectations(tmp_path): + """`assert` with an empty expectation set would report success having checked nothing.""" + res = _run(f"{SCRIPT} assert --always", _env(tmp_path)) + assert res.returncode != 0