timothy
2fd798cccf
#649 — the enforced review-verdict.yml guard had drifted strictly WEAKER than the advisory merge-consent hook: four rounds of #643 hardening landed on the copy whose failures produce a human prompt, and never reached the copy that writes the branch-protection-required review-verdict/h10 status. Its fail-closed behaviour on a garbage response was also incidental (an empty `n` erroring a bash conditional to false), not designed. Extract scripts/pr-changed-files.sh as the single implementation both call. Shared MECHANISM, not policy: the two docs-only allow-lists differ deliberately and stay separate. review-verdict.yml now checks out the BASE ref, never the PR head, so a PR cannot rewrite the gate that judges it. #648 — baking jq into docker/ci/Dockerfile provably cannot cover the gate that broke: review-verdict.yml is runs-on:small with no toolchain pin, so it gets the host's jq 1.6 (checked, not assumed). Add scripts/jq-preflight.sh: floor+observable everywhere, and a --expect tripwire on script-tests only — pinning the required merge check would deadlock every merge on a jq bump. Verified by mutation: six guards individually broken, each turning exactly its own test red, then restored byte-identical. fixes #648 fixes #649
145 lines
5.9 KiB
Python
145 lines
5.9 KiB
Python
"""Tests for `scripts/jq-preflight.sh` — the jq version contract (ersatztv#648).
|
|
|
|
The axis this guards. Every shell gate in this repo is authored on a Mac shipping jq 1.8.x; the CI
|
|
runner ships jq 1.6. Nothing pinned or checked that, and three independent divergences surfaced in a
|
|
single day — `jq -e` over empty input (exit 4 vs 0), `contains("<NUL>")` (false vs true for every
|
|
string), and the parse-error exit code (5 vs 4, colliding with "no output"). Each was patched with a
|
|
version-stable construct, but patching constructs one at a time leaves the AXIS untested.
|
|
|
|
These tests shim `jq` on PATH with a fake reporting an arbitrary version, so the preflight's own
|
|
behaviour is verified by MEASUREMENT rather than by observing a green CI tick — ersatztv#648's third
|
|
Done-when box. Doing it here rather than by pushing a deliberately-red commit also keeps the proof
|
|
reproducible: it re-runs on every PR instead of living in one CI run's history.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = REPO_ROOT / "scripts" / "jq-preflight.sh"
|
|
WORKFLOWS = REPO_ROOT / ".gitea" / "workflows"
|
|
# Resolved BEFORE PATH is narrowed to the shim dir — the tests strip PATH down to just that
|
|
# directory, so `bash` could not be found by name from inside them.
|
|
BASH = shutil.which("bash") or "/bin/bash"
|
|
|
|
|
|
@pytest.fixture
|
|
def preflight(tmp_path):
|
|
bindir = tmp_path / "bin"
|
|
bindir.mkdir()
|
|
|
|
class Handle:
|
|
def with_jq(self, version_line):
|
|
"""Install a fake `jq` reporting `version_line` for --version."""
|
|
shim = bindir / "jq"
|
|
shim.write_text("#!/bin/sh\n"
|
|
'if [ "$1" = "--version" ]; then echo "%s"; exit 0; fi\nexit 0\n'
|
|
% version_line)
|
|
shim.chmod(0o755)
|
|
|
|
def without_jq(self):
|
|
shim = bindir / "jq"
|
|
if shim.exists():
|
|
shim.unlink()
|
|
|
|
def run(self, *args):
|
|
env = dict(os.environ)
|
|
# PATH contains ONLY the shim dir. An earlier draft appended /usr/bin:/bin "for the
|
|
# basics" and the missing-jq test passed vacuously against the developer machine's real
|
|
# /usr/bin/jq — the negative case was never negative. The script needs nothing from PATH
|
|
# but jq itself (`command -v` is a builtin, and bash is invoked by absolute path), so
|
|
# there is nothing to keep.
|
|
env["PATH"] = str(bindir)
|
|
return subprocess.run([BASH, str(SCRIPT), *args],
|
|
env=env, capture_output=True, text=True)
|
|
|
|
return Handle()
|
|
|
|
|
|
def test_the_version_is_printed_so_the_job_log_shows_it(preflight):
|
|
"""ersatztv#648's second Done-when box: the jq version CI actually uses must be OBSERVABLE."""
|
|
preflight.with_jq("jq-1.6")
|
|
r = preflight.run()
|
|
assert r.returncode == 0, r.stderr
|
|
assert "jq-1.6" in r.stdout
|
|
|
|
|
|
def test_floor_mode_accepts_the_runner_version(preflight):
|
|
preflight.with_jq("jq-1.6")
|
|
assert preflight.run().returncode == 0
|
|
|
|
|
|
def test_floor_mode_accepts_a_newer_jq(preflight):
|
|
"""No upper bound in floor mode — review-verdict.yml writes the REQUIRED merge check, so a jq
|
|
bump must never be able to deadlock every merge in the repo."""
|
|
preflight.with_jq("jq-1.8.2")
|
|
assert preflight.run().returncode == 0
|
|
|
|
|
|
def test_below_the_floor_is_LOUD(preflight):
|
|
preflight.with_jq("jq-1.5")
|
|
r = preflight.run()
|
|
assert r.returncode == 1
|
|
assert "below the supported floor" in r.stderr.lower()
|
|
|
|
|
|
def test_missing_jq_is_loud(preflight):
|
|
preflight.without_jq()
|
|
r = preflight.run()
|
|
assert r.returncode == 1
|
|
assert "not on PATH" in r.stderr
|
|
|
|
|
|
@pytest.mark.parametrize("version_line", ["jq-1.6-dirty", "jq-1.6", "jq-1.6.0"])
|
|
def test_build_suffixes_still_parse_as_1_6(preflight, version_line):
|
|
"""A packaging suffix must not fail a perfectly ordinary jq closed — that would be a tripwire
|
|
firing on noise, which is how tripwires get disabled."""
|
|
preflight.with_jq(version_line)
|
|
assert preflight.run("--expect", "1.6").returncode == 0, version_line
|
|
|
|
|
|
def test_expect_mismatch_is_LOUD(preflight):
|
|
"""THE TRIPWIRE. scripts/tests exercises the jq 1.6 path only because the runner ships 1.6. If
|
|
the runner were upgraded that coverage would vanish silently, so the pin must go red instead."""
|
|
preflight.with_jq("jq-1.7.1")
|
|
r = preflight.run("--expect", "1.6")
|
|
assert r.returncode == 1
|
|
assert "expected jq 1.6, found 1.7" in r.stderr
|
|
|
|
|
|
def test_expect_match_passes(preflight):
|
|
preflight.with_jq("jq-1.6")
|
|
assert preflight.run("--expect", "1.6").returncode == 0
|
|
|
|
|
|
def test_unknown_argument_is_a_usage_error(preflight):
|
|
preflight.with_jq("jq-1.6")
|
|
assert preflight.run("--pin", "1.6").returncode == 2
|
|
|
|
|
|
# --- Wiring guards: the preflight is worthless if a caller silently stops running it ------------
|
|
|
|
def test_script_tests_pins_and_review_verdict_only_floors():
|
|
"""The asymmetry is deliberate and load-bearing, so it is asserted rather than merely commented.
|
|
|
|
`script-tests` carries `--expect` (a tripwire on a normal job). `review-verdict.yml` must NOT:
|
|
it writes the branch-protection-required `review-verdict/h10` status, so a hard version pin
|
|
there would turn a jq bump on the runner into a repo-wide merge deadlock.
|
|
"""
|
|
pr_checks = (WORKFLOWS / "pr-checks.yml").read_text()
|
|
review_verdict = (WORKFLOWS / "review-verdict.yml").read_text()
|
|
|
|
assert "jq-preflight.sh --expect" in pr_checks, \
|
|
"script-tests must pin the jq version — that pin is the tripwire"
|
|
assert "jq-preflight.sh" in review_verdict, \
|
|
"review-verdict.yml must at least print/floor-check its jq version"
|
|
assert "jq-preflight.sh --expect" not in review_verdict, \
|
|
("review-verdict.yml must NOT pin a jq version: it writes the required review-verdict/h10 "
|
|
"status, so a pin would deadlock every merge on a jq bump (ersatztv#648)")
|