From 8d9be0d4b8a18869d62167e4d5c1d58294dfffc2 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sun, 16 Aug 2026 10:57:44 +0200 Subject: [PATCH] fix(794): a status of 1 must come from pytest, not from a failed cd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2: BLOCKED, 1 Blocker + 2 High + 2 Low; 5 of 10 round-1 items verified fixed. THE BLOCKER IS THE SAFETY CLAIM ITSELF. `( cd X && pytest ); rc=$?` returns the SUBSHELL's status, so a failed cd or a failed redirection also yields 1 — and the "only pytest exit 1 counts as red" rule then accepts it as a witnessed test failure with pytest never having run. The rule I added last round to close a false green had a false green underneath it. Fixed structurally: the phase writes pytest's status to a marker file only AFTER pytest returns. An absent marker means "pytest did not complete" and is refused as a harness failure, which cannot be confused with a verdict. Proven load-bearing — test_a_harness_failure_is_NOT_reported_as_PROVEN goes RED when the marker logic is mutated back to trusting the subshell status. Two fail-open enumerations, both real: - the CI job read each commit's trailer with an unchecked command substitution, so a failing `git show` produced an empty selector, skipped the commit, and let the job exit 0 having been unable to inspect a possible claim; - `while ... done < <(git diff-tree ...)` cannot see the producer's exit status, so a git failure that had already emitted one complete record passed the non-empty check and reverted only PART of the commit — overstating "without the fix". The stream is now captured to a file with a checked status, and any truncated record is fatal. The selector is now split once into an array and always expanded as "${SEL[@]}". Unquoted it also applied PATHNAME expansion, so a node id containing a glob character could select different tests than the trailer names. PROVE_FIX_PATH is now gated behind PROVE_FIX_MUTATION_RUN=1: as an unrestricted ambient override, a stray CI value would silently point the whole suite at another script. Verified: 14 tests; the blocker fix and the mutation proof each witnessed red against their own mutants; real commits 84165ab75 PROVEN / UNPROVEN against an unrelated test; scripts/tests 681 passed, 2 skipped. Proves: scripts/tests/test_prove_fix.py Co-Authored-By: Claude Opus 5 (1M context) --- .gitea/workflows/pr-checks.yml | 12 +++++- scripts/prove-fix.sh | 76 ++++++++++++++++++++++++++------- scripts/tests/test_prove_fix.py | 37 +++++++++++++++- 3 files changed, 107 insertions(+), 18 deletions(-) diff --git a/.gitea/workflows/pr-checks.yml b/.gitea/workflows/pr-checks.yml index d14822779..906b5d145 100644 --- a/.gitea/workflows/pr-checks.yml +++ b/.gitea/workflows/pr-checks.yml @@ -257,8 +257,16 @@ jobs: # Trim whitespace only — NOT `xargs`, which applies quote parsing and turns a # legitimate parametrised node id like test_x[can't] into an empty selector, # silently dropping a real claim. - sel="$(git show -s --format='%(trailers:key=Proves,valueonly)' "$sha" | head -1 \ - | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + # Extract with a CHECKED status. `sel="$(git show ... )"` under `set -uo + # pipefail` but no `-e` yields an empty selector when git fails, the commit is + # skipped, and the job exits 0 having been unable to inspect a possible claim — + # fail-open in the step that decides what gets checked. + if ! raw="$(git show -s --format='%(trailers:key=Proves,valueonly)' "$sha")"; then + echo "::error::git show failed for $sha — cannot read its trailers, so this" \ + "job cannot assert anything about it. Refusing to pass." + exit 1 + fi + sel="$(printf '%s\n' "$raw" | head -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" [ -n "$sel" ] || continue claimed=$((claimed + 1)) diff --git a/scripts/prove-fix.sh b/scripts/prove-fix.sh index c3b1bca1c..e61ae0fb6 100755 --- a/scripts/prove-fix.sh +++ b/scripts/prove-fix.sh @@ -101,12 +101,20 @@ if [ -z "$SELECTOR" ]; then SELECTOR="${SELECTOR#"${SELECTOR%%[![:space:]]*}"}" SELECTOR="${SELECTOR%"${SELECTOR##*[![:space:]]}"}" fi +# Split ONCE, deliberately, into an array — then always expand as "${SEL[@]}". Leaving the +# scalar unquoted at the call site would also apply PATHNAME EXPANSION, so a node id +# containing a glob character could select different tests than the trailer names. +read -r -a SEL <<< "$SELECTOR" +set +f + [ -n "$SELECTOR" ] || die \ "no test selector: pass one, or give the commit a 'Proves: ' trailer. Refusing to guess from the touched test files — that silently proves nothing when a fix edits an existing test, which is the failure mode this script exists to prevent." 3 # ---------------------------------------------------------------- classify +TMP_ENUM="$(mktemp -d "${TMPDIR:-/tmp}/prove-fix-enum.XXXXXX")" || die "mktemp failed" 5 +trap 'rm -rf "$TMP_ENUM" >/dev/null 2>&1; cleanup' EXIT # NUL-delimited --name-status: git tells us whether each path was Added, Modified, # Deleted or Renamed. Deriving added-ness from git's own status letter beats probing the # parent blob and interpreting the error text — `cat-file -e parent:new.py` fails with a @@ -114,16 +122,25 @@ edits an existing test, which is the failure mode this script exists to prevent. # both mis-classifies that case and would leave real git failures indistinguishable. # -z also protects paths containing newlines/quotes, which git otherwise quotes and which # would then silently stay un-reverted while we claim to have reverted them. +# Capture to a file with a CHECKED status first. Inside `done < <(git ...)` the producer's +# exit status is unavailable, so a git failure that had already emitted one complete record +# would pass the non-empty check and revert only PART of the commit — overstating "without +# the fix" and manufacturing a proof from an incomplete revert. +NS="$TMP_ENUM/name-status" +git -C "$REPO" diff-tree --no-commit-id -r -z --name-status "$SHA" >"$NS" \ + || die "git diff-tree failed for $SHA — cannot enumerate what to revert" 5 + CHANGED=(); STATUS=() while IFS= read -r -d '' st; do - IFS= read -r -d '' path || break + IFS= read -r -d '' path || die "truncated --name-status stream after status '$st'" 5 case "$st" in - R*|C*) IFS= read -r -d '' newpath || break # rename/copy: NUL NUL - CHANGED+=("$newpath"); STATUS+=("A") # the new path did not exist in the parent - CHANGED+=("$path"); STATUS+=("D") ;; # the old path did + R*|C*) IFS= read -r -d '' newpath \ + || die "truncated rename/copy record after '$st' '$path'" 5 + CHANGED+=("$newpath"); STATUS+=("A") # the new path is absent in the parent + CHANGED+=("$path"); STATUS+=("D") ;; # the old path is present in it *) CHANGED+=("$path"); STATUS+=("${st:0:1}") ;; esac -done < <(git -C "$REPO" diff-tree --no-commit-id -r -z --name-status "$SHA") +done <"$NS" [ "${#CHANGED[@]}" -gt 0 ] || die "commit touches no files, or enumeration failed: $SHA" 5 is_test_path() { @@ -158,17 +175,34 @@ printf 'reverting %d non-test file(s), keeping %d test file(s)\n' \ "${#NON_TEST[@]}" "${#TEST_FILES[@]}" TMP="$(mktemp -d "${TMPDIR:-/tmp}/prove-fix.XXXXXX")" || die "mktemp failed" 5 -mkdir -p "$TMP/tmp-control" "$TMP/tmp-reverted" +mkdir -p "$TMP/tmp-control" "$TMP/tmp-reverted" || die "could not create phase temp dirs" 5 # Each phase gets its OWN worktree and TMPDIR, and pytest's cache is disabled. Sharing one # worktree lets state written during the control run decide the second run: a test that -# creates a marker file and fails when it already exists would "go red" with the fix still -# in place — a false PROVEN produced entirely by the harness. -run_phase() { # $1 = worktree, $2 = label; returns pytest's exit code - ( cd "$1" \ - && PYTHONPATH=. TMPDIR="$TMP/tmp-$2" PYTEST_ADDOPTS= \ - python3 -m pytest $SELECTOR -q --continue-on-collection-errors -p no:cacheprovider \ - >"$TMP/out-$2.txt" 2>&1 ) +# creates a marker and fails when it already exists would "go red" with the fix still in +# place — a false PROVEN manufactured entirely by the harness. +# +# PYTEST_RC IS READ FROM A MARKER, NOT FROM THE SUBSHELL. `( cd X && pytest ) ; rc=$?` +# returns the SUBSHELL's status, and a failed `cd` or a failed redirection also yields 1 — +# which the "only exit 1 is red" rule would then accept as a witnessed test failure with +# pytest never having run. Cold review found exactly that. The marker file is written only +# after pytest RETURNS, so its absence means "pytest did not complete" and can never be +# mistaken for a verdict. +PYTEST_RC="" +run_phase() { # $1 = worktree, $2 = label; sets PYTEST_RC or returns non-zero + rm -f "$TMP/rc-$2" + ( + cd "$1" || exit 91 + exec >"$TMP/out-$2.txt" 2>&1 || exit 92 + PYTHONPATH=. TMPDIR="$TMP/tmp-$2" PYTEST_ADDOPTS= \ + python3 -m pytest "${SEL[@]}" -q --continue-on-collection-errors -p no:cacheprovider + printf '%s' "$?" > "$TMP/rc-$2" + ) + if [ ! -s "$TMP/rc-$2" ]; then + return 1 # pytest never completed — harness failure, NOT a test result + fi + PYTEST_RC="$(cat "$TMP/rc-$2")" + return 0 } # ---------------------------------------------------------------- control @@ -176,7 +210,13 @@ git -C "$REPO" worktree add --detach "$TMP/wt-control" "$SHA" >/dev/null 2>&1 \ || die "could not create the control worktree at $SHA" 5 echo "--- control: running the selector WITH the fix ---" -run_phase "$TMP/wt-control" control; CONTROL_RC=$? +if ! run_phase "$TMP/wt-control" control; then + sed -n '1,25p' "$TMP/out-control.txt" 2>/dev/null >&2 + die "the control run did not complete: pytest produced no exit status. +This is a harness failure (bad cd, unwritable log, missing interpreter), not a test +result, and must never be reported as a verdict." 5 +fi +CONTROL_RC="$PYTEST_RC" echo "--- control pytest exit: $CONTROL_RC ---" if [ "$CONTROL_RC" -ne 0 ]; then @@ -207,7 +247,13 @@ for i in "${!NON_TEST[@]}"; do done echo "--- running the selector WITHOUT the fix ---" -run_phase "$TMP/wt-reverted" reverted; RC=$? +if ! run_phase "$TMP/wt-reverted" reverted; then + sed -n '1,25p' "$TMP/out-reverted.txt" 2>/dev/null >&2 + die "the reverted run did not complete: pytest produced no exit status. +This is a harness failure, not a witnessed red. Accepting it would let a broken cd or an +unwritable log masquerade as proof — the defect this script exists to catch." 5 +fi +RC="$PYTEST_RC" echo "--- pytest exit: $RC ---" # ---------------------------------------------------------------- verdict diff --git a/scripts/tests/test_prove_fix.py b/scripts/tests/test_prove_fix.py index 783e000ab..96513944b 100644 --- a/scripts/tests/test_prove_fix.py +++ b/scripts/tests/test_prove_fix.py @@ -24,6 +24,14 @@ REPO_ROOT = Path(__file__).resolve().parents[2] # Overridable so a MUTANT copy can be driven through these very tests. That is what makes # the mutation proof below a witnessed red rather than an argument: the unchanged test is # re-run against the mutant and observed to fail. +# Overridable so a MUTANT copy can be driven through these very tests — that is what makes +# the mutation proof below a witnessed red rather than an argument. Gated behind a sentinel +# so a stray CI value cannot silently point the whole suite at another script. +if os.environ.get("PROVE_FIX_PATH") and os.environ.get("PROVE_FIX_MUTATION_RUN") != "1": + raise RuntimeError( + "PROVE_FIX_PATH is set without PROVE_FIX_MUTATION_RUN=1. That would silently test a " + "different script than the one this suite vouches for." + ) PROVE_FIX = Path(os.environ.get("PROVE_FIX_PATH") or (REPO_ROOT / "scripts" / "prove-fix.sh")) @@ -238,7 +246,7 @@ def test_MUTATION_disarming_the_UNPROVEN_clause_reddens_the_refusal_test( mutant.write_text(src[: src.index(marker)] + src[end:]) assert marker not in mutant.read_text(), "mutation did not remove the clause" - env = {**os.environ, "PROVE_FIX_PATH": str(mutant)} + env = {**os.environ, "PROVE_FIX_PATH": str(mutant), "PROVE_FIX_MUTATION_RUN": "1"} nested = subprocess.run( ["python3", "-m", "pytest", f"{Path(__file__).name}::test_unrelated_test_is_UNPROVEN", "-q", "-p", "no:cacheprovider"], @@ -293,3 +301,30 @@ def test_SIGTERM_mid_run_never_reports_PROVEN(tmp_path: Path) -> None: assert proc.returncode != 0, f"a signalled run must exit non-zero, got {proc.returncode}" assert "PROVEN" not in out, f"a signalled run must not print a verdict:\n{out}" + + +def test_a_harness_failure_is_NOT_reported_as_PROVEN(fixrepo: Path, tmp_path: Path) -> None: + """pytest must actually RUN for its status to mean anything. + + `( cd X && pytest ); rc=$?` returns the SUBSHELL's status, so a failed cd or a failed + redirection also yields 1 — which "only exit 1 is red" would accept as a witnessed test + failure with pytest never having run. Cold review found that; the script now reads + pytest's status from a marker written only after it returns. + + Simulated by making python3 unavailable to the run: the phase cannot produce a status, + so the script must refuse rather than infer. + """ + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + (fake_bin / "python3").write_text("#!/bin/sh\nexit 127\n") + (fake_bin / "python3").chmod(0o755) + + env = {**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"} + r = subprocess.run( + ["bash", str(PROVE_FIX), "--repo", str(fixrepo), "HEAD", "scripts/tests/test_add.py"], + capture_output=True, text=True, cwd=str(fixrepo), env=env, + ) + assert "PROVEN" not in r.stdout, ( + f"a run where pytest could not execute must not produce a verdict:\n{r.stdout}\n{r.stderr}" + ) + assert r.returncode != 0, f"expected a refusal, got {r.returncode}"