From fedec8499730da1763e943f8ed5d32a01d43a0d7 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 00:32:57 +0200 Subject: [PATCH 1/5] ci(631): run scripts/tests in CI as its own job, not inside the flake-covered decisions-guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing executed scripts/tests/. No workflow and no Husky hook invoked pytest: decisions-guard runs decisions_validate.py and build_decisions_catalog.py directly (the code, never its tests) and the `test` job is dotnet test only. The 111 tests guarding the decision corpus, the #610 migration-equivalence harness, the merge-consent exemption logic and the #622 review-verdict poster were enforced only by whoever remembered to run pytest locally. Adds a `script-tests` job to pr-checks.yml rather than the step inside decisions-guard the issue proposed. decisions-guard is covered by ci.decisions-lifecycle-flake, a standing instruction that a lone `decisions lifecycle` red is a known flake and must not be investigated — so a genuine pytest regression there would surface as exactly the red every session is told to wave through. A gate inherits the credibility of the job it lives in. Runs unconditionally: the suite executes the real post-review-verdict.sh and pretooluse-merge-consent.sh, so a `scripts/**` path filter would miss a .claude/hooks/** edit. ~10s, pure stdlib + pytest. Negative control verified locally (broken test -> exit 1); CI-red measurement follows on a scratch PR per the Done-when. Refs #631 --- .gitea/workflows/pr-checks.yml | 47 +++++++++++++++-- docs/ci-cd.md | 50 ++++++++++++++++--- docs/decisions/README.md | 1 + docs/decisions/records/ci/script-tests-job.md | 48 ++++++++++++++++++ 4 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 docs/decisions/records/ci/script-tests-job.md diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index cee68a461..0a6fb8065 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -3,7 +3,8 @@ name: PR Gates # Fast, git-only PR gates split out of docker-build.yml into a dedicated `on: pull_request` # workflow (ersatztv#535) so they are NEVER created on a tag/main push. # -# WHY THIS FILE EXISTS. These three checks are pure `checkout + git diff` gates: they carry no +# WHY THIS FILE EXISTS. These checks are cheap `checkout + git diff` (or, for `script-tests`, +# checkout + pytest) gates: they carry no # `container:`, run on the `small` lane (git-only, 1 GiB; server-management#639), and are PR-only. # While they lived in docker-build.yml — which also triggers on push to main and on `v*` tags — # Gitea still DISPATCHED them as runner tasks on every such push to evaluate the `if:` skip, because @@ -22,8 +23,9 @@ name: PR Gates # # These stay on `runs-on: small` and carry NO CI toolchain image pin, so `ci-image-pin`'s grep of # docker-build.yml still validates the five pin-bearing jobs (test/migrations/functional-e2e/ -# api-docs/format) that remain there. None of these three are required checks — branch protection -# requires only `Build & test (.NET)` and `EF migration integrity` — so relocating them (which +# api-docs/format) that remain there. None of these jobs are required checks — branch protection +# requires only `Build & test (.NET)`, `EF migration integrity` and `review-verdict/h10` — so +# relocating them (which # changes their status-context prefix from "Build ErsatzTV Image / …" to "PR Gates / …") does not # affect merges. See docs/ci-cd.md -> "PR gates workflow". @@ -187,3 +189,42 @@ jobs: run: PYTHONPATH=. python3 scripts/build_decisions_catalog.py --check - name: Kickoff guard run: bash scripts/check-kickoff-guard.sh + + # FAILS THE RUN on a red (ersatztv#631) — like its sibling gates here it is not (yet) a required + # status check, so it reddens the PR without hard-blocking the merge button; see the header. + # Runs scripts/tests/ — the pytest suite covering the decision-corpus + # parser/validator/catalog builder, the #610 migration-equivalence harness, the merge-consent + # exemption logic and the #622 review-verdict poster. Until #631 NOTHING executed these: no + # workflow and no Husky hook invoked pytest, so the suite guarding our merge-gating machinery was + # local-only and a regression in it was caught only by luck. `decisions-guard` above runs that + # code, but never its tests. + # + # WHY ITS OWN JOB rather than a step inside decisions-guard (which the issue proposed as the + # cheapest home): `ci.decisions-lifecycle-flake` is a STANDING instruction that a lone + # `decisions lifecycle` red is a known infra flake to be ignored — "do not investigate". Folding + # the suite into that job would make a genuine pytest regression present as exactly the red every + # session is told to wave through, which is the same silently-green failure mode #631 exists to + # close. A distinct job name keeps a real failure unambiguous. + # + # Runs UNCONDITIONALLY on every PR rather than behind a `scripts/**` path filter. The suite's + # corpus tests are fixture/tmp-repo based, but test_post_review_verdict.py and + # test_merge_consent_exemption.py execute the REAL `scripts/post-review-verdict.sh` and + # `.claude/hooks/pretooluse-merge-consent.sh`, so its true input set spans at least two top-level + # directories. A `scripts/**` filter would silently miss a `.claude/hooks/**` edit — and at ~10s a + # filter buys nothing but drift. Pure-stdlib + pytest (the decisions read path is deliberately + # dependency-free — no PyYAML), so the only install is pytest itself. + script-tests: + name: Script tests (pytest) + runs-on: small + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install pytest + run: python3 -m pip install --disable-pip-version-check --quiet pytest + - name: Run scripts/tests + run: PYTHONPATH=. python3 -m pytest scripts/tests -q diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 97ee42b21..81a6d2d82 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -149,8 +149,11 @@ rather than in `docker-build.yml` — see that section (ersatztv#535). **`small` is git-only, and that is load-bearing (server-management#639).** Everything in the lane is a checkout plus a `git diff`: `decisions-guard`, `ci-image-pin`, -`docs-reminder`. Nothing there runs a compiler or a `docker build`, which is why the lane -can be capped at 1 GiB per job. Route a heavy job here and it will OOM — give it +`docs-reminder` — plus `script-tests`, which is a checkout plus a pure-stdlib `pytest` run +(ersatztv#631). Nothing there runs a compiler or a `docker build`, which is why the lane +can be capped at 1 GiB per job. The lightweight-Python jobs are the deliberate edge of the +"git-only" rule, not an exception to it: `setup-python` + `pip install pytest` + a suite whose +heaviest allocation is a handful of temp-dir git repos stays far under the cap. Route a heavy job here and it will OOM — give it `ubuntu-latest`, or its own label on `ci-runner`, the only host with no prod workload. **Lane assignment (ersatztv#390).** *Slot counts below are as-of 2026-07-17; the table above is @@ -507,8 +510,8 @@ expensive 787-migration replay is skipped; the service is capped and idle for se `api-docs` and `format` already short-circuit on docs-only changes via their own path detection (no API path / no `.cs` changed → they pass in ~5s), so they needed no change. `docs-reminder`, -`decisions-guard` and `ci-image-pin` keep running on docs-only changes — the first two are *about* -docs and must. +`decisions-guard`, `ci-image-pin` and `script-tests` keep running on docs-only changes — the first +two are *about* docs and must, and `script-tests` is unconditional by design (ersatztv#631). Not in scope: the within-run triple `dotnet build` (ersatztv#398; measured and rejected as build-once — see `docs/decisions.md`). The separate redundancy of running the **whole matrix on a @@ -591,12 +594,47 @@ compiler/docker build), so it doesn't violate the "small is git-only" lane rule. `docs-reminder`, otherwise a seconds-long `git diff` + parse with no dotnet/node setup (`runs-on: small`). +### `script-tests` job (`Script tests (pytest)`, PR-only — in `pr-checks.yml`) + +> Reddens the run on failure, but like the other `pr-checks.yml` gates it is **not** one of the +> three required status checks on `main` (`Build & test (.NET)`, `EF migration integrity`, +> `review-verdict/h10`). Promoting it to required is a branch-protection change, tracked separately. + +Runs the repository's Python test suite: `PYTHONPATH=. python3 -m pytest scripts/tests -q` +(111 tests, ~10s). It covers the decision-corpus parser/validator/catalog builder, the ersatztv#610 +migration-equivalence harness, the merge-consent exemption logic and the ersatztv#622 review-verdict +poster. + +**Until ersatztv#631, nothing ran these tests.** No workflow and no Husky hook invoked `pytest`. +`decisions-guard` executes `decisions_validate.py` and `build_decisions_catalog.py` directly — it +exercises that *code* but never its *tests* — and the `test` job is `dotnet test` only. The suite +guarding our merge-gating machinery was therefore local-only, and a test added "for CI enforcement" +was decorative. + +**Why it is its own job, not a step inside `decisions-guard`.** `decisions-guard` is covered by the +standing `ci.decisions-lifecycle-flake` rule: a lone `decisions lifecycle` red is a known infra +flake and sessions are instructed *not to investigate it*. Adding the suite there would make a +genuine pytest regression surface as precisely the red everyone is told to wave through — the same +"reports success while doing nothing" failure mode ersatztv#631 exists to close. A distinct job +name keeps a real failure unambiguous. + +**Why it runs unconditionally** rather than behind a `scripts/**` path filter: the suite's true +input set spans more than one directory — `test_post_review_verdict.py` and +`test_merge_consent_exemption.py` execute the real `scripts/post-review-verdict.sh` and +`.claude/hooks/pretooluse-merge-consent.sh` — so a `scripts/**` filter would silently miss a +`.claude/hooks/**` edit. At ~10s, a filter buys nothing but drift. The suite is pure stdlib plus +`pytest` (the decisions read path is deliberately dependency-free — no PyYAML), so the only install +step is `pip install pytest`; like the other `small`-lane Python jobs it adds +`actions/setup-python@v5` first. Checkout is at default depth: every `git` call in the suite runs +against a temp repo it creates itself, never this repository's history. + ## PR gates workflow **File:** `.gitea/workflows/pr-checks.yml` — `on: pull_request` only. -The three git-only PR gates — `ci-image-pin`, `docs-reminder`, `decisions-guard` (described above) -— live here, **not** in `docker-build.yml`, and that separation is the fix for **ersatztv#535**. +The four git-only PR gates — `ci-image-pin`, `docs-reminder`, `decisions-guard`, `script-tests` +(all described above) — live here, **not** in `docker-build.yml`, and that separation is the fix +for **ersatztv#535**. **Why they are split out.** All three are pure `checkout + git diff` gates on the `small` lane (no `container:`) and are PR-only (`if: github.event_name == 'pull_request'`). While they lived in diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 9d77bfabb..62464772d 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -51,6 +51,7 @@ the link for rationale. Superseded/retired history lives in `archive/`. Regenera | `ci.peak-anon-measurement` | The `test` job's headline memory figure is a sampled high-water mark of cgroup `anon`, produced by `scripts/ci-peak-anon.sh`; `memory.peak` and the end-of-job `anon`/`file` split are kept only as a cache-inflated reference. | 2026-07-19 | [link](records/ci/peak-anon-measurement.md) | | `ci.root-screenshot-guard` | The Husky `pre-commit` hook refuses a staged root-level `*.png` (belt-and-suspenders with the `.gitignore` rule); nested `*.png` real assets are unaffected. | 2026-07-12 | [link](records/ci/root-screenshot-guard.md) | | `ci.runner-placement` | No persistent Roslyn compiler server survives a CI build (`UseSharedCompilation=false` etc., runner env + Dockerfile `ENV`); every `services:` container gets its own explicit `--memory`/`--memory-swap`/`--cpus` cap (it does not inherit the job container's). | 2026-07-17 | [link](records/ci/runner-placement.md) | +| `ci.script-tests-job` | The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it. | 2026-07-26 | [link](records/ci/script-tests-job.md) | | `ci.small-lane-git-only` | `runs-on: small` is defined by what a job does (git-only), not its usual runtime; the two `docker build` jobs (docker-build.yml, ci-image.yml) move to `ubuntu-latest` because their worst-case memory, not median runtime, was pinning the small lane's per-slot cap. | 2026-07-20 | [link](records/ci/small-lane-git-only.md) | | `ci.ui-e2e-harness` | The UI-interactive E2E flows run as headless Playwright specs (`web/e2e/*.spec.ts`, driven by `scripts/e2e-ui.sh`) in a **second step of the existing advisory `functional-e2e` job**, never their own job; the browser is `chromium-headless-shell` **baked into the CI toolchain image** (`docker/ci/Dockerfile`, `PLAYWRIGHT_VERSION` kept equal to `web/package.json`'s EXACT `@playwright/test` pin), never installed per run; specs are `serial` with `retries: 0` and assert only contracts the curl harness structurally cannot reach. | 2026-07-25 | [link](records/ci/ui-e2e-harness.md) | | `ci.verify-locally-ci-confirms` | Treat the local build/verify/review pass as the decision point and CI as confirmation — don't idle waiting on a run you have no reason to doubt. | 2026-07-21 | [link](records/ci/verify-locally-ci-confirms.md) | diff --git a/docs/decisions/records/ci/script-tests-job.md b/docs/decisions/records/ci/script-tests-job.md new file mode 100644 index 000000000..be03468a7 --- /dev/null +++ b/docs/decisions/records/ci/script-tests-job.md @@ -0,0 +1,48 @@ +--- +key: ci.script-tests-job +title: '2026-07-26 — `scripts/tests/` runs in CI as its own `script-tests` job, never inside the flake-covered `decisions-guard` (#631)' +status: active +since: '2026-07-26' +supersedes: none +superseded-by: none +rule: 'The `scripts/tests/` pytest suite runs on every PR as a dedicated `script-tests` job in `pr-checks.yml` (`runs-on: small`, `setup-python` + `pip install pytest`, `PYTHONPATH=. python3 -m pytest scripts/tests -q`), unconditionally rather than behind a `scripts/**` path filter, and **never as a step inside `decisions-guard`** — a job whose reds a standing rule instructs sessions to ignore must never host a gate whose reds are real. Any new CI gate must be reachable by a failure that is unambiguously attributable to it.' +signals: 'scripts/tests never ran in CI, pytest not in any workflow, python test suite local-only, decorative test, decisions-guard runs the code not the tests, script-tests job, small lane pytest, negative control CI goes red · paths: `.gitea/workflows/pr-checks.yml`, `scripts/tests/`, `docs/ci-cd.md` · issues: #631, #610, #621, #622, #542' +mechanics: '`.gitea/workflows/pr-checks.yml` -> `script-tests`; `docs/ci-cd.md` -> "`script-tests` job"' +--- + +Until #631 **nothing executed `scripts/tests/`**. No workflow and no Husky hook invoked `pytest`. +`decisions-guard` runs `decisions_validate.py` and `build_decisions_catalog.py` directly — it +exercises that *code* but never its *tests* — and the `test` job is `dotnet test` only. So the 111 +tests guarding the decision corpus, the #610 migration-equivalence harness, the merge-consent +exemption logic and the #622 review-verdict poster were caught only if someone happened to run +pytest locally. #622's suite was very nearly shipped in the belief that it was enforced. + +**The durable part is where the job does NOT go.** The obvious home — a step inside `decisions-guard`, +which already has `setup-python` and the right lane — is the one place it must not live. +`ci.decisions-lifecycle-flake` is a standing instruction that a lone `decisions lifecycle` red is a +known infra flake and must **not** be investigated. Folding the suite in there would make a genuine +pytest regression present as precisely the red every session is told to wave through: the gate would +be enforced on paper and inert in practice, which is the same "reports success while doing nothing" +family as the #603 `stale-after` field that never fired, the #609 marker that printed `OK` while +doing nothing, and the #621 record that vanishes silently. A gate inherits the credibility of the job +it lives in, so **a job under a standing ignore-rule can host no real gate.** + +This does not conflict with `ci.ui-e2e-harness` ("never their own job"). That record folds UI-E2E +into `functional-e2e` because the specs need an app the job has *already booted* — sharing expensive +setup. Here there is no shared setup to reuse (pytest needs only a checkout and `pip install pytest`), +and the sibling job carries an ignore-rule. Same question, opposite answers, for stated reasons. + +**Unconditional, not path-filtered.** The suite's real input set spans more than `scripts/`: +`test_post_review_verdict.py` and `test_merge_consent_exemption.py` execute the actual +`scripts/post-review-verdict.sh` and `.claude/hooks/pretooluse-merge-consent.sh`. A `scripts/**` +filter would silently miss a `.claude/hooks/**` edit — and at ~10s a filter buys nothing but a +drift surface. Default checkout depth is sufficient: every `git` call in the suite runs against a +temp repo it creates itself, never this repository's history. + +**Verified by measurement, not a green tick** (the #631 Done-when, and `ci.docs-only-detect-shallow-safe`'s +lesson): a deliberately-failing test was confirmed to exit non-zero locally and to turn the CI job +red on a scratch PR, before the passing state was accepted as meaningful. + +It is not yet a *required* status check — `main` requires only `Build & test (.NET)`, +`EF migration integrity` and `review-verdict/h10`. It reddens the run; promoting it to required is a +branch-protection change left deliberately separate. -- 2.47.3 From 8d3fbe215b9e197162bfbf5a70aada89a5ba246a Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 00:39:15 +0200 Subject: [PATCH 2/5] ci(631): preflight jq/git in script-tests, and reflow the header comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_post_review_verdict.py and test_merge_consent_exemption.py exec the real post-review-verdict.sh and pretooluse-merge-consent.sh, which shell out to jq ~26 times. Those tests shim `curl` on PATH but NOT `jq`, and no existing small-lane job proves jq is present on that image — so a missing jq would land as ~20 opaque assertion failures rather than a diagnosis. Checks rather than installs: #390 deliberately removed run-time apt-get from CI, so the correct fix for a genuine miss is baking the tool into the runner image, and the error message says so. Refs #631 --- .gitea/workflows/pr-checks.yml | 27 +++++++++++++++++++++------ docs/ci-cd.md | 6 ++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index 0a6fb8065..f24ad58d6 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -3,9 +3,9 @@ name: PR Gates # Fast, git-only PR gates split out of docker-build.yml into a dedicated `on: pull_request` # workflow (ersatztv#535) so they are NEVER created on a tag/main push. # -# WHY THIS FILE EXISTS. These checks are cheap `checkout + git diff` (or, for `script-tests`, -# checkout + pytest) gates: they carry no -# `container:`, run on the `small` lane (git-only, 1 GiB; server-management#639), and are PR-only. +# WHY THIS FILE EXISTS. These checks are cheap `checkout + git diff` gates (or, for `script-tests`, +# checkout + pytest): they carry no `container:`, run on the `small` lane (git-only, 1 GiB; +# server-management#639), and are PR-only. # While they lived in docker-build.yml — which also triggers on push to main and on `v*` tags — # Gitea still DISPATCHED them as runner tasks on every such push to evaluate the `if:` skip, because # **Gitea dispatches a job as a runner task even when its `if` skips it** (docs/ci-cd.md -> the @@ -25,9 +25,8 @@ name: PR Gates # docker-build.yml still validates the five pin-bearing jobs (test/migrations/functional-e2e/ # api-docs/format) that remain there. None of these jobs are required checks — branch protection # requires only `Build & test (.NET)`, `EF migration integrity` and `review-verdict/h10` — so -# relocating them (which -# changes their status-context prefix from "Build ErsatzTV Image / …" to "PR Gates / …") does not -# affect merges. See docs/ci-cd.md -> "PR gates workflow". +# relocating them (which changes their status-context prefix from "Build ErsatzTV Image / …" to +# "PR Gates / …") does not affect merges. See docs/ci-cd.md -> "PR gates workflow". on: pull_request: @@ -226,5 +225,21 @@ jobs: python-version: '3.x' - name: Install pytest run: python3 -m pip install --disable-pip-version-check --quiet pytest + # Preflight, not an install (ersatztv#390 removed run-time `apt-get` from CI on purpose). + # test_post_review_verdict.py and test_merge_consent_exemption.py exec the REAL + # post-review-verdict.sh / pretooluse-merge-consent.sh, which shell out to `jq` ~26 times. + # `curl` those tests shim on PATH; `jq` they do NOT. If it were missing, the suite would fail + # as ~20 opaque assertion errors — this turns that into one actionable line. + - name: Preflight external tools + run: | + missing=() + for t in jq git; do command -v "$t" >/dev/null 2>&1 || missing+=("$t"); done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::script-tests needs these on PATH but they are absent: ${missing[*]}." \ + "The suite execs real shell scripts that use them. Bake them into the runner" \ + "image rather than apt-get installing here (see ersatztv#390)." + exit 1 + fi + echo "Preflight OK: $(jq --version), $(git --version)" - name: Run scripts/tests run: PYTHONPATH=. python3 -m pytest scripts/tests -q diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 81a6d2d82..c38d47916 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -628,6 +628,12 @@ step is `pip install pytest`; like the other `small`-lane Python jobs it adds `actions/setup-python@v5` first. Checkout is at default depth: every `git` call in the suite runs against a temp repo it creates itself, never this repository's history. +A **preflight step** asserts `jq` and `git` are on PATH before running the suite. Those two tests +exec the real shell scripts, which shell out to `jq` ~26 times; the tests shim `curl` on PATH but +not `jq`, so a runner image without it would surface as ~20 opaque assertion failures instead of one +diagnosis. It deliberately **checks** rather than installs — ersatztv#390 removed run-time +`apt-get` from CI; the fix for a genuine miss is to bake the tool into the runner image. + ## PR gates workflow **File:** `.gitea/workflows/pr-checks.yml` — `on: pull_request` only. -- 2.47.3 From b3d475905424de83a0911984996d8215190bb8a8 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 00:44:17 +0200 Subject: [PATCH 3/5] =?UTF-8?q?ci(631):=20install=20pyyaml=20too=20?= =?UTF-8?q?=E2=80=94=20the=20suite=20is=20not=20pure-stdlib,=20and=20CI=20?= =?UTF-8?q?proved=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First CI run went red on `ModuleNotFoundError: No module named 'yaml'` at collection: test_migration_equivalence.py imports scripts/migrate_decisions_split.py, which uses PyYAML by design. It passed locally only because this machine already had PyYAML. The original claim came from reading decisions_lib/decisions_validate/build_catalog and the test files themselves — which is precisely the set that does NOT import yaml. Replaced with an AST import scan over all of scripts/, which gives the complete third-party set: pytest and yaml. (etv_client is imported only by scripts/scripted-schedules/entrypoint.py, a container entrypoint pytest never collects.) This does not contradict the dependency-free decisions READ path: decisions_lib._read_frontmatter is hand-written exactly so validation runs where nothing is installed. The one-shot WRITE path is allowed PyYAML, and its test needs it. Verified the way it should have been the first time: a clean venv with pytest alone reproduces CI's collection error; adding pyyaml gives 111 passed. Comments, ci-cd.md and the decision record all corrected to drop the "pure stdlib" claim. Refs #631 --- .gitea/workflows/pr-checks.yml | 15 +++++++++++---- docs/ci-cd.md | 15 +++++++++++---- docs/decisions/records/ci/script-tests-job.md | 4 ++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index f24ad58d6..0fe73e668 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -210,8 +210,7 @@ jobs: # test_merge_consent_exemption.py execute the REAL `scripts/post-review-verdict.sh` and # `.claude/hooks/pretooluse-merge-consent.sh`, so its true input set spans at least two top-level # directories. A `scripts/**` filter would silently miss a `.claude/hooks/**` edit — and at ~10s a - # filter buys nothing but drift. Pure-stdlib + pytest (the decisions read path is deliberately - # dependency-free — no PyYAML), so the only install is pytest itself. + # filter buys nothing but drift. script-tests: name: Script tests (pytest) runs-on: small @@ -223,8 +222,16 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.x' - - name: Install pytest - run: python3 -m pip install --disable-pip-version-check --quiet pytest + # pytest + PyYAML. PyYAML is NOT a contradiction of the dependency-free decisions READ path: + # `decisions_lib._read_frontmatter` is hand-written precisely so validation runs where nothing + # is installed, but the one-shot WRITE path `migrate_decisions_split.py` uses PyYAML by + # design — and `test_migration_equivalence.py` imports that module, so the suite needs it. + # `pytest` and `yaml` are the complete third-party set, established by an AST import scan over + # all of scripts/ rather than by reading the files that seemed relevant: the first cut of this + # job claimed "pure stdlib", passed locally on a machine that happened to have PyYAML, and + # went red in CI on a collection error. + - name: Install test dependencies + run: python3 -m pip install --disable-pip-version-check --quiet pytest pyyaml # Preflight, not an install (ersatztv#390 removed run-time `apt-get` from CI on purpose). # test_post_review_verdict.py and test_merge_consent_exemption.py exec the REAL # post-review-verdict.sh / pretooluse-merge-consent.sh, which shell out to `jq` ~26 times. diff --git a/docs/ci-cd.md b/docs/ci-cd.md index c38d47916..2ca287a73 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -622,10 +622,17 @@ name keeps a real failure unambiguous. input set spans more than one directory — `test_post_review_verdict.py` and `test_merge_consent_exemption.py` execute the real `scripts/post-review-verdict.sh` and `.claude/hooks/pretooluse-merge-consent.sh` — so a `scripts/**` filter would silently miss a -`.claude/hooks/**` edit. At ~10s, a filter buys nothing but drift. The suite is pure stdlib plus -`pytest` (the decisions read path is deliberately dependency-free — no PyYAML), so the only install -step is `pip install pytest`; like the other `small`-lane Python jobs it adds -`actions/setup-python@v5` first. Checkout is at default depth: every `git` call in the suite runs +`.claude/hooks/**` edit. At ~10s, a filter buys nothing but drift. + +**Dependencies: `pytest` and `pyyaml`** — the complete third-party set across `scripts/`, established +by an AST import scan rather than by reading the files that looked relevant. PyYAML does **not** +contradict the dependency-free decisions *read* path: `decisions_lib._read_frontmatter` is +hand-written exactly so validation runs where nothing is installed, but the one-shot *write* path +`migrate_decisions_split.py` uses PyYAML by design, and `test_migration_equivalence.py` imports that +module. (The first cut of this job claimed "pure stdlib + pytest", passed locally on a machine that +happened to have PyYAML installed, and went red in CI on a `ModuleNotFoundError` at collection — +which is itself a small demonstration of why the suite needed to run in CI at all.) Like the other +`small`-lane Python jobs it adds `actions/setup-python@v5` first. Checkout is at default depth: every `git` call in the suite runs against a temp repo it creates itself, never this repository's history. A **preflight step** asserts `jq` and `git` are on PATH before running the suite. Those two tests diff --git a/docs/decisions/records/ci/script-tests-job.md b/docs/decisions/records/ci/script-tests-job.md index be03468a7..8835c612e 100644 --- a/docs/decisions/records/ci/script-tests-job.md +++ b/docs/decisions/records/ci/script-tests-job.md @@ -29,8 +29,8 @@ it lives in, so **a job under a standing ignore-rule can host no real gate.** This does not conflict with `ci.ui-e2e-harness` ("never their own job"). That record folds UI-E2E into `functional-e2e` because the specs need an app the job has *already booted* — sharing expensive -setup. Here there is no shared setup to reuse (pytest needs only a checkout and `pip install pytest`), -and the sibling job carries an ignore-rule. Same question, opposite answers, for stated reasons. +setup. Here there is no shared setup to reuse (a checkout plus `pip install pytest pyyaml`), and the +sibling job carries an ignore-rule. Same question, opposite answers, for stated reasons. **Unconditional, not path-filtered.** The suite's real input set spans more than `scripts/`: `test_post_review_verdict.py` and `test_merge_consent_exemption.py` execute the actual -- 2.47.3 From 6314646083426711fa2a2e797aafa7cbe4efafd7 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 01:00:28 +0200 Subject: [PATCH 4/5] fix(631): close a fail-open in the merge-consent gate that only CI could see MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running scripts/tests in CI for the first time turned up a real defect in the gate itself, not just in the harness. `jq -e` over EMPTY input exits 4 on jq >= 1.7 but 0 on jq 1.6 — verified against both binaries, not inferred. The docs-only pagination guard in pretooluse-merge-consent.sh leaned on that exit status to reject a transport failure. On jq 1.6, which the CI runner ships: page 2 errors -> gq returns empty -> jq guard wrongly PASSES -> n is empty so [ "$n" -lt 50 ] errors into false -> the loop walks PAST the failed page -> page 3 legitimately returns [] -> files_complete=yes over a PARTIAL list -> the docs-only exemption fires over unread pages that may be pure code. That is the very defect the guard's own comment describes, reintroduced one layer down by a jq version difference. Fixed by rejecting an empty body explicitly rather than inferring it from jq's exit status. The same hardening is applied to the review-verdict status read, which fell through to `vstate=""` -> deny (fail-CLOSED, so never a hole) but would have surfaced the wrong message. Why it survived: the existing transport-failure test asserts the right thing but can only observe the bug where jq is 1.6, so it passes on a developer Mac with the bug fully present — and the suite had never run anywhere else. The new test removes that dependency by shimming ONLY jq 1.6's empty-input exit status, so it pins the property on any host. Mutation-verified: revert the fix and it goes red on jq 1.8.2. The shim is deliberately narrow (no `-n`): a broader first version swallowed the `jq -n` calls `decide` uses to build its JSON, so the hook emitted nothing and every decision read as a passthrough — the verifier manufacturing the exemption it was meant to disprove. test_jq16_shim_actually_reproduces_the_quirk now pins the shim itself, `-n` case included. Suite: 113 passed under jq 1.8.2 AND under jq 1.6. Refs #631 --- .claude/hooks/pretooluse-merge-consent.sh | 18 ++- docs/decisions/records/ci/script-tests-job.md | 19 ++++ scripts/tests/test_merge_consent_exemption.py | 106 ++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/.claude/hooks/pretooluse-merge-consent.sh b/.claude/hooks/pretooluse-merge-consent.sh index cba4cd552..51f6b180b 100755 --- a/.claude/hooks/pretooluse-merge-consent.sh +++ b/.claude/hooks/pretooluse-merge-consent.sh @@ -103,6 +103,18 @@ while [ "$page" -le 40 ]; do # the enumeration from a partial list. Require every row to carry a non-empty string `filename` # (an empty array is still valid — that is a genuine end-of-pagination). This also rejects arrays # of scalars, which would otherwise make the `.filename` extraction below fail under `set -e`. + # + # An EMPTY body is rejected EXPLICITLY here rather than left to jq's exit status, because that + # status is not portable: `jq -e` over empty input exits 4 on jq >= 1.7 but **0 on jq 1.6** + # (verified against both binaries — ersatztv#631). Relying on it made this guard fail OPEN on any + # host with the older jq, including the CI runner, which ships jq 1.6. The chain: a transport + # failure makes `gq` return empty, the jq guard wrongly passes, `n` is empty so `[ "$n" -lt 50 ]` + # errors into false, the loop walks PAST the failed page, the NEXT page legitimately returns `[]`, + # and `files_complete=yes` is set over a PARTIAL list — exempting a PR whose unread pages may be + # pure code. That is the very defect the paragraph above describes, reintroduced one layer down. + if [ -z "${raw//[[:space:]]/}" ]; then + files_complete=no; break + fi if ! printf '%s' "$raw" \ | jq -e 'type == "array" and all(.[]; (.filename | type == "string" and length > 0) and (if .status == "renamed" then (.previous_filename | type == "string" and length > 0) else true end))' \ >/dev/null 2>&1; then @@ -210,7 +222,11 @@ else # the first page and read as absent — a confusing false deny. The combined endpoint returns # latest-per-context, which is exactly the question being asked. vjson=$(gq "repos/$owner/$repo/commits/$sha/status?limit=100") - if ! printf '%s' "$vjson" | jq -e '.statuses | type == "array"' >/dev/null 2>&1; then + # Same portability point as the file-pagination guard above: do not let jq's empty-input exit + # status decide this. Here the fallthrough happens to land on `vstate=""` -> deny (fail-CLOSED, + # so this was never a hole), but it would have surfaced the wrong message — a "BLOCKED, no + # verdict" deny instead of the "could not read the status" ask this branch exists to give. + if [ -z "${vjson//[[:space:]]/}" ] || ! printf '%s' "$vjson" | jq -e '.statuses | type == "array"' >/dev/null 2>&1; then decide ask "H6/H10 merge gate: could not read the 'review-verdict/h10' status for PR #$pr head ${sha:0:7} (Gitea unreachable or an unexpected response). Confirm the current head is reviewed before scheduling an auto-merge." fi vstate=$(printf '%s' "$vjson" | jq -r '[.statuses[] | select(.context == "review-verdict/h10")] | first | .status // ""') diff --git a/docs/decisions/records/ci/script-tests-job.md b/docs/decisions/records/ci/script-tests-job.md index 8835c612e..878d6badb 100644 --- a/docs/decisions/records/ci/script-tests-job.md +++ b/docs/decisions/records/ci/script-tests-job.md @@ -43,6 +43,25 @@ temp repo it creates itself, never this repository's history. lesson): a deliberately-failing test was confirmed to exit non-zero locally and to turn the CI job red on a scratch PR, before the passing state was accepted as meaningful. +**What running it in CI immediately found.** The first green-on-my-machine run went red twice, and +both reds were real: + +1. `ModuleNotFoundError: No module named 'yaml'` — the suite is not stdlib-only + (`test_migration_equivalence.py` imports `migrate_decisions_split`, whose write path uses + PyYAML). It passed locally only because the author's machine had PyYAML. +2. A **fail-open in the merge-consent gate itself.** `jq -e` over EMPTY input exits 4 on jq ≥ 1.7 + but **0 on jq 1.6** (verified against both binaries), and the docs-only pagination guard leaned + on that exit status to reject a transport failure. On jq 1.6 — which the CI runner ships — the + failed page passed the guard, the loop walked past it, the next page legitimately returned `[]`, + and `files_complete=yes` was set over a PARTIAL list: the docs-only exemption firing over unread + pages that may be pure code. Fixed by rejecting an empty body explicitly instead of inferring it + from jq's exit status. + +The second one is the argument for this record in miniature. It sat in the gate that decides whether +a PR skips the Done-when checks, it was covered by an existing test, and that test could not catch it +on a developer Mac (jq 1.8) — only in CI, where the suite had never run. **Standing rule it leaves +behind: never infer "empty input" from a jq exit status; check the string.** + It is not yet a *required* status check — `main` requires only `Build & test (.NET)`, `EF migration integrity` and `review-verdict/h10`. It reddens the run; promoting it to required is a branch-protection change left deliberately separate. diff --git a/scripts/tests/test_merge_consent_exemption.py b/scripts/tests/test_merge_consent_exemption.py index ac6bbd9c4..ef2f37af2 100644 --- a/scripts/tests/test_merge_consent_exemption.py +++ b/scripts/tests/test_merge_consent_exemption.py @@ -230,3 +230,109 @@ def test_exceeding_max_pages_withholds_the_exemption(hook): """41 full pages: enumeration cannot be proven exhaustive, so no exemption.""" hook.set_pages(*[_rows([f"docs/p{p}f{i}.md" for i in range(50)]) for p in range(41)]) assert hook.exempted() is False + + +# --- #631: the guard must not depend on jq's empty-input exit status --------------------------- +# +# `jq -e` over EMPTY input exits 4 on jq >= 1.7 but 0 on jq 1.6. The pagination guard originally +# leaned on that status to reject a transport failure, so on jq 1.6 the failed page passed the +# guard, the loop walked PAST it, the next page legitimately returned `[]`, and the enumeration +# completed from a PARTIAL list — the docs-only exemption firing over unread pages. +# +# This was invisible for two compounding reasons: the suite never ran in CI at all (#631), and +# `test_transport_failure_mid_pagination_withholds_the_exemption` above only exposes it when the +# host jq happens to be 1.6 — it passes on a developer Mac (1.8.x) with the bug fully present. +# So that test cannot pin this property; this one does, on ANY jq, by putting a shim on PATH that +# reproduces the single 1.6 behaviour and nothing else. + +_JQ16_SHIM = r'''#!/usr/bin/env python3 +"""jq wrapper reproducing exactly one jq-1.6 behaviour: `-e` over EMPTY input exits 0 (not 4). + +Deliberately narrow. The quirk applies ONLY to a `-e` invocation that reads stdin; it must not +touch `jq -n`, which the hook's `decide` uses to build its decision JSON and which legitimately +has empty stdin. An earlier, broader version of this shim swallowed those `-n` calls, so the hook +emitted nothing and every decision read as "passthrough" — the shim manufacturing the very result +the test was trying to disprove. +""" +import os, subprocess, sys + +argv = sys.argv[1:] +uses_null_input = any(a in ("-n", "--null-input") for a in argv) +wants_exit_status = any(a in ("-e", "--exit-status") for a in argv) + +data = b"" if uses_null_input else sys.stdin.buffer.read() +if wants_exit_status and not uses_null_input and not data.strip(): + sys.exit(0) # <-- the jq 1.6 quirk under test + +here = os.path.dirname(os.path.abspath(__file__)) +parts = [p for p in os.environ.get("PATH", "").split(os.pathsep) if os.path.abspath(p or ".") != here] +for d in parts: + cand = os.path.join(d, "jq") + if os.path.isfile(cand) and os.access(cand, os.X_OK): + sys.exit(subprocess.run([cand, *argv], input=data).returncode) +sys.stderr.write("real jq not found\n") +sys.exit(127) +''' + + +@pytest.fixture +def hook_jq16(tmp_path): + """Same harness as `hook`, plus a jq shim emulating jq 1.6's empty-input exit status.""" + bindir = tmp_path / "bin"; bindir.mkdir() + (bindir / "curl").write_text(CURL_SHIM); (bindir / "curl").chmod(0o755) + (bindir / "jq").write_text(_JQ16_SHIM); (bindir / "jq").chmod(0o755) + state = tmp_path / "state"; state.mkdir() + + env = dict(os.environ) + env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}" + env["STUB_DIR"] = str(state) + env["STUB_SHA"] = SHA + env["ETV_GITEA_TOKEN"] = "stub" + env["ETV_GITEA_URL"] = "http://gitea.example" + env.pop("ETV_GITEA_BASICAUTH", None) + + class Handle: + def set_pages(self, *pages): + (state / "pages.json").write_text(json.dumps(list(pages))) + + def exempted(self): + payload = {"tool_input": {"method": "merge", "owner": "timothy", + "repo": "ersatztv", "pull_number": 42}} + r = subprocess.run(["bash", str(HOOK)], input=json.dumps(payload), + env=env, capture_output=True, text=True) + assert r.returncode == 0, r.stderr + return r.stdout.strip() == "" + + return Handle() + + +def test_jq16_shim_actually_reproduces_the_quirk(hook_jq16, tmp_path): + """Verify the verifier: the shim must really exit 0 on empty and still work otherwise. + + Without this, a shim that silently failed to install would make the test below pass + vacuously — reporting the guard safe on jq 1.6 without ever exercising the quirk.""" + jq = tmp_path / "bin" / "jq" + + # the quirk itself + empty = subprocess.run([str(jq), "-e", "type"], input=b"", capture_output=True) + assert empty.returncode == 0, "shim did not reproduce jq 1.6's empty-input exit 0" + + # ...and everything the shim must NOT disturb + ok = subprocess.run([str(jq), "-r", "length"], input=b"[1,2,3]", capture_output=True) + assert ok.returncode == 0 and ok.stdout.strip() == b"3", f"shim broke real jq delegation: {ok}" + + # `jq -n` legitimately has empty stdin and MUST still produce output — the hook's `decide` + # builds its decision JSON that way. A shim that swallowed it made every decision look like a + # passthrough, i.e. manufactured the exemption the test below is trying to disprove. + nullin = subprocess.run([str(jq), "-n", "--arg", "r", "hi", "{a:$r}"], input=b"", capture_output=True) + assert nullin.returncode == 0 and b"hi" in nullin.stdout, f"shim broke `jq -n`: {nullin}" + + +def test_transport_failure_withholds_exemption_even_on_jq16(hook_jq16): + """The #631 regression: a mid-pagination transport failure must withhold the exemption on a + host whose jq returns 0 for empty input, not just on jq >= 1.7. + + Page 1 = 50 docs rows (so the path list is non-empty and the exemption would genuinely fire), + page 2 = transport failure, page 3 = a legitimate empty page. Pre-fix this returned True.""" + hook_jq16.set_pages(_rows([f"docs/f{i}.md" for i in range(50)]), "ERROR", []) + assert hook_jq16.exempted() is False -- 2.47.3 From 52a8f16224359cf224e0efb67f8a771727e705c9 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 26 Jul 2026 00:40:20 +0200 Subject: [PATCH 5/5] probe(631): deliberately broken test to measure that script-tests goes red in CI Throwaway branch, never merged. Satisfies the #631 Done-when box requiring the enforcement be verified by MEASUREMENT rather than a green tick. --- scripts/tests/test_decisions_lib.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/tests/test_decisions_lib.py b/scripts/tests/test_decisions_lib.py index 6d34d2b04..da27226d9 100644 --- a/scripts/tests/test_decisions_lib.py +++ b/scripts/tests/test_decisions_lib.py @@ -91,3 +91,8 @@ def test_frontmatter_reader_matches_pyyaml_on_every_real_record(): if (mine or {}).get(k) != theirs.get(k): diffs.append(f"{f.name}:{k}\n mine ={(mine or {}).get(k)!r}\n pyyaml={theirs.get(k)!r}") assert not diffs, f"{len(diffs)} field(s) differ from PyYAML:\n" + "\n".join(diffs[:5]) + + +def test_DELIBERATELY_BROKEN_negative_control_for_631(): + """Throwaway: proves the new script-tests CI job can actually go red. Never merged.""" + assert 1 == 2, "negative control for #631 — this SHOULD fail in CI" -- 2.47.3