"""Tests for `scripts/jq-preflight.sh` — the jq version contract (ersatztv#648). The axis this guards. Every shell gate in this repo is authored on a Mac shipping jq 1.8.x; the CI runner ships jq 1.6. Nothing pinned or checked that, and three independent divergences surfaced in a single day — `jq -e` over empty input (exit 4 vs 0), `contains("")` (false vs true for every string), and the parse-error exit code (5 vs 4, colliding with "no output"). Each was patched with a version-stable construct, but patching constructs one at a time leaves the AXIS untested. These tests shim `jq` on PATH with a fake reporting an arbitrary version, so the preflight's own behaviour is verified by MEASUREMENT rather than by observing a green CI tick — ersatztv#648's third Done-when box. Doing it here rather than by pushing a deliberately-red commit also keeps the proof reproducible: it re-runs on every PR instead of living in one CI run's history. """ from __future__ import annotations import os import shutil import subprocess from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parents[2] SCRIPT = REPO_ROOT / "scripts" / "jq-preflight.sh" WORKFLOWS = REPO_ROOT / ".gitea" / "workflows" # Resolved BEFORE PATH is narrowed to the shim dir — the tests strip PATH down to just that # directory, so `bash` could not be found by name from inside them. BASH = shutil.which("bash") or "/bin/bash" def _shq(s): """Single-quote a string for /bin/sh.""" return "'" + s.replace("'", "'\\''") + "'" @pytest.fixture def preflight(tmp_path): bindir = tmp_path / "bin" bindir.mkdir() class Handle: def with_jq(self, version_line, stderr="", exit_code=0): """Install a fake `jq` reporting `version_line` for --version. `stderr` and `exit_code` exist because a shim that ALWAYS exits 0 and never writes to stderr structurally cannot observe the worst failure this script has: a jq that cannot start. The preflight was folding stderr into the parse via `2>&1` and discarding the exit status, so a glibc-mismatch message containing `2.34` parsed as version 2.34 and PASSED the floor. Every case such a shim can express is clean, so every test passes. """ shim = bindir / "jq" body = '#!/bin/sh\nif [ "$1" = "--version" ]; then\n' if version_line: body += f' printf "%s\\n" {_shq(version_line)}\n' if stderr: body += f' printf "%s\\n" {_shq(stderr)} >&2\n' body += f" exit {exit_code}\nfi\nexit 0\n" shim.write_text(body) shim.chmod(0o755) def without_jq(self): shim = bindir / "jq" if shim.exists(): shim.unlink() def run(self, *args): env = dict(os.environ) # PATH contains ONLY the shim dir. Appending /usr/bin:/bin "for the basics" makes the # missing-jq test pass vacuously against the developer machine's real /usr/bin/jq — the # negative case is never negative. The script needs nothing from PATH but jq itself # (`command -v` is a builtin, and bash is invoked by absolute path), so there is nothing # to keep. env["PATH"] = str(bindir) return subprocess.run([BASH, str(SCRIPT), *args], env=env, capture_output=True, text=True) def run_bytes(self, *args): """Same, but WITHOUT text mode. `text=True` enables universal-newlines translation, which rewrites `\\r` to `\\n` in the captured output — so any assertion about a stray carriage return is unfalsifiable through `run()`. That is not hypothetical: the CR test passed identically with the strip removed until this was noticed, while the mutant demonstrably emits `... = jq-1.6\\r (parsed 1.6; ...)` at the byte level. """ env = dict(os.environ) env["PATH"] = str(bindir) return subprocess.run([BASH, str(SCRIPT), *args], env=env, capture_output=True) return Handle() def test_the_version_is_printed_so_the_job_log_shows_it(preflight): """ersatztv#648's second Done-when box: the jq version CI actually uses must be OBSERVABLE.""" preflight.with_jq("jq-1.6") r = preflight.run() assert r.returncode == 0, r.stderr assert "jq-1.6" in r.stdout def test_floor_mode_accepts_the_runner_version(preflight): preflight.with_jq("jq-1.6") assert preflight.run().returncode == 0 def test_floor_mode_accepts_a_newer_jq(preflight): """No upper bound in floor mode — review-verdict.yml writes the REQUIRED merge check, so a jq bump must never be able to deadlock every merge in the repo.""" preflight.with_jq("jq-1.8.2") assert preflight.run().returncode == 0 def test_below_the_floor_is_LOUD(preflight): preflight.with_jq("jq-1.5") r = preflight.run() assert r.returncode == 1 assert "below the supported floor" in r.stderr.lower() def test_missing_jq_is_loud(preflight): preflight.without_jq() r = preflight.run() assert r.returncode == 1 assert "not on PATH" in r.stderr @pytest.mark.parametrize("version_line", ["jq-1.6-dirty", "jq-1.6", "jq-1.6.0"]) def test_build_suffixes_still_parse_as_1_6(preflight, version_line): """A packaging suffix must not fail a perfectly ordinary jq closed — that would be a tripwire firing on noise, which is how tripwires get disabled.""" preflight.with_jq(version_line) assert preflight.run("--expect", "1.6").returncode == 0, version_line def test_expect_mismatch_is_LOUD(preflight): """THE TRIPWIRE. scripts/tests exercises the jq 1.6 path only because the runner ships 1.6. If the runner were upgraded that coverage would vanish silently, so the pin must go red instead.""" preflight.with_jq("jq-1.7.1") r = preflight.run("--expect", "1.6") assert r.returncode == 1 assert "expected jq 1.6, found 1.7" in r.stderr def test_expect_match_passes(preflight): preflight.with_jq("jq-1.6") assert preflight.run("--expect", "1.6").returncode == 0 def test_unknown_argument_is_a_usage_error(preflight): preflight.with_jq("jq-1.6") assert preflight.run("--pin", "1.6").returncode == 2 def test_expect_without_a_value_is_a_usage_error_WITH_output(preflight): """`shift 2` on a missing value exits 1 under `set -e` with NOTHING on either stream. A CI step that dies with an empty log is the diagnostic hole this script exists to remove.""" preflight.with_jq("jq-1.6") r = preflight.run("--expect") assert r.returncode == 2 assert "requires a value" in r.stderr # --- Version parsing: the guard must never assert a floor against an unparsed version ---------- # # The original strip-based parse assumed the format is exactly `jq-X.Y`. Anything else left major or # minor EMPTY, and the sanity check concatenated them — so `jq version 1.6` produced "6", which is # non-empty and all-digits, so the check PASSED. The floor comparison then ran `[ "" -lt 1 ]`, which # errors; `set -e` exempts a failing command in an `if` condition, so the conditional read false and # the script exited 0 having asserted NOTHING. That is this script's own stated failure mode, # reproduced inside itself, which is why these cases are pinned rather than left to inspection. @pytest.mark.parametrize( "version_line", [ "jq version 1.6", # some distro wrappers print this form "JQ-1.6", "jq-1.6-dirty", ], ) def test_unusual_but_parseable_version_forms_are_accepted(preflight, version_line): preflight.with_jq(version_line) r = preflight.run() assert r.returncode == 0, f"{version_line!r}: {r.stderr}" assert "parsed 1.6" in r.stdout @pytest.mark.parametrize("version_line", ["jq-1.-6", "jq-.6", "not-a-version", ""]) def test_unparseable_version_fails_CLOSED_rather_than_asserting_nothing(preflight, version_line): preflight.with_jq(version_line) r = preflight.run() assert r.returncode == 1, ( f"{version_line!r} exited {r.returncode}: an unparsed version must never reach — or " "silently skip — the floor assertion" ) assert "could not parse" in r.stderr def test_a_jq_that_cannot_START_fails_closed(preflight): """THE case the previous shim could not express, and the guard therefore got wrong. A jq broken by a glibc mismatch (the canonical post-base-image-bump failure) exits 127 and writes `... version 'GLIBC_2.34' not found` to STDERR. The preflight was reading `jq --version 2>&1` and discarding the exit status, so that message became the parse input, `2.34` matched, and the floor was certified green on a jq that cannot run at all. """ preflight.with_jq("", stderr="jq: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found", exit_code=127) r = preflight.run() assert r.returncode == 1 assert "cannot run" in r.stderr assert "parsed 2.34" not in r.stdout, "stderr must never be parsed as a version" @pytest.mark.parametrize( "version_line", [ "warning: something 3.14", # a noise line carrying a plausible number "2026.07.26 jq-1.6", # a date prefix, which outranks the real version if unanchored "jq-master-v0.0.0-1.6", ], ) def test_a_number_that_is_not_the_VERSION_is_not_accepted_as_one(preflight, version_line): """Matching the first `.` ANYWHERE let a prefix win over the real version. `2026.07.26 jq-1.6` parsed as 2026.07 and sailed over the floor. The pattern is anchored to the leading `jq` token, so these fail closed instead.""" preflight.with_jq(version_line) r = preflight.run() assert r.returncode == 1, f"{version_line!r} was accepted as a version" assert "could not parse" in r.stderr @pytest.mark.parametrize( "version_line", [ "jq-99999999999999999999999.0", "jq-1.99999999999999999999999", ], ) def test_an_OUT_OF_RANGE_digit_run_fails_closed(preflight, version_line): """The original fail-open mechanism, resurrected via an over-long number. A regex that guarantees *digits* does not guarantee they fit `test`'s integer range. With a 23-digit major, `[ "$major" -lt "$min_major" ]` errors with "integer expression expected" — and `set -e` exempts a failing command in an `if` condition, so the conditional read false and THE FLOOR WAS NEVER ASSERTED, exit 0. Identical in shape to the empty-string case that started this. Bounding the run with `{1,9}` alone was NOT enough either: the pattern is unanchored at the end, so an over-long minor just matched its first 9 digits and compared that instead — a mis-parse that passes. The trailing non-digit requirement is what actually closes it. """ preflight.with_jq(version_line) r = preflight.run() assert r.returncode == 1, f"{version_line!r} exited 0 — the floor was not asserted" @pytest.mark.parametrize( "version_line", [ # Killed by the SEPARATOR restriction (a blank separator must be followed by `version`). "jq\n2.34: cannot load shared library", "jq\n\n\n99.9", "jq -- 2.34 (real jq-1.6)", "jq\t\t9.9", # Killed ONLY by the first-line slice + `[[:blank:]]`. These carry the literal word `version`, # so the separator restriction is satisfied and cannot save us — the newline must be excluded # from the separator class AND the parse confined to line one. # # Without these, mutation measured that reverting BOTH of those changes together # (`[[:blank:]]`→`[[:space:]]` and parsing `$raw` instead of `$first`) left the whole suite # GREEN: the four cases above are all killed by the separator alone, so they attributed the fix # to the wrong layer. "jq\nversion\n9.9", "jq\nversion 9.9", "jq \n version \n 9.9", ], ) def test_a_number_AFTER_the_jq_token_is_not_reachable_across_filler(preflight, version_line): """Two independent layers keep a stray number from being read as the version, and both are pinned here: the separator must be one of the forms real jq emits (`jq-1.6` / `jq version 1.6`), AND the match is confined to the first line with `[[:blank:]]` (which, unlike `[[:space:]]`, does not match a newline). An 'anchor' with neither parses `jq\\n2.34: cannot load` as 2.34.""" preflight.with_jq(version_line) r = preflight.run() assert r.returncode == 1, f"{version_line!r} was accepted as a version" def test_a_CRLF_version_line_parses_and_logs_without_the_carriage_return(preflight): """The trailing `\\r` strip was unpinned — the commit claimed CRLF was verified, but nothing in the suite contained one. Harmless today (a `\\r` satisfies the trailing non-digit boundary, so the version still parses) but the log line would carry a stray CR.""" preflight.with_jq("jq-1.6\r") r = preflight.run_bytes() assert r.returncode == 0, r.stderr assert b"parsed 1.6" in r.stdout # Two separate traps had to be cleared for this assertion to mean anything: # 1. `str.splitlines()` also splits on `\r`, so inspecting the "version in use" line would drop # the stray CR before the assertion could see it; # 2. `subprocess.run(text=True)` translates `\r` to `\n` outright, so even raw-string checks on # `r.stdout` were unfalsifiable. # Both made the test pass identically with the strip removed. Hence `run_bytes()` and a bytes # comparison — verified by mutation, not by reading the code. assert b"\r" not in r.stdout, "the carriage return leaked into the log line" def test_the_observability_line_stays_on_ONE_line(preflight): """The no-arg mode exists to put a single grep-able version line in the job log; interpolating a multi-line `--version` would split it.""" preflight.with_jq("jq-1.6\ntrailing noise") r = preflight.run() assert r.returncode == 0, r.stderr version_lines = [ln for ln in r.stdout.splitlines() if "version in use" in ln] assert len(version_lines) == 1 assert "trailing noise" not in r.stdout @pytest.mark.parametrize( "version_line,expected", [ ("jq-1.6 (Debian 1.6-2.1)", "1.6"), # distro packaging suffix ("jq-1.10", "1.10"), # two-digit minor: must compare numerically, not lexically ("jq-1.7.1", "1.7"), ("jq-1.6.0", "1.6"), ("jq-v1.6", "1.6"), ("JQ-1.6", "1.6"), ], ) def test_legitimate_forms_still_parse_to_the_right_version(preflight, version_line, expected): preflight.with_jq(version_line) r = preflight.run() assert r.returncode == 0, f"{version_line!r}: {r.stderr}" assert f"parsed {expected}" in r.stdout # --- Wiring guards: the preflight is worthless if a caller silently stops running it ------------ def test_script_tests_pins_the_jq_version(): """The pin is the tripwire, so its presence is asserted rather than merely commented. SCOPE NOTE — the symmetric assertion about `review-verdict.yml` (that it runs the FLOOR-only mode and must never pin, because it writes the branch-protection-required `review-verdict/h10` status and a pin would deadlock every merge on a jq bump) lands with the follow-up PR that wires that workflow. It cannot land here: that workflow checks out the BASE ref, and the base is `main`, which does not yet contain `scripts/jq-preflight.sh`. """ pr_checks = (WORKFLOWS / "pr-checks.yml").read_text() assert "jq-preflight.sh --expect" in pr_checks, "script-tests must pin the jq version — that pin is the tripwire" def test_review_verdict_never_pins_a_jq_version(): """Whatever else changes, the REQUIRED merge check must never carry a hard version pin. Asserted now, before the workflow is wired, so the constraint is already enforced when the follow-up PR adds the floor-only call — rather than being a comment someone can miss. """ review_verdict = (WORKFLOWS / "review-verdict.yml").read_text() assert "jq-preflight.sh --expect" not in review_verdict, ( "review-verdict.yml must NOT pin a jq version: it writes the required review-verdict/h10 " "status, so a pin would deadlock every merge on a jq bump (ersatztv#648)" )