ersatztv/scripts/prove-fix.sh

#!/usr/bin/env bash
# prove-fix.sh — witness a fix's test failing BEFORE the fix (ersatztv#794).
#
# THE RULE THIS ENFORCES. `testing.guard-ships-with-mutation-proof` says a guard is not
# tested because a test involving it passes; it ships with a proof it can fail. The same
# argument applies to every bug fix, and nothing enforced it. #776 and #793 each took six
# review rounds, and in both the recurring mechanism was one thing: a fix's test was
# written to confirm the fix, not to discriminate against its absence.
#
# HOW. Run the named tests at the commit (CONTROL — must be GREEN), then again in a
# SEPARATE fresh worktree with the commit's non-test files reverted (must be RED). The
# control is what makes the second run mean anything: a test that is already failing
# proves nothing by failing again.
#
# ONLY pytest exit 1 COUNTS AS RED, and that is the whole safety argument. "Non-zero" is
# not "the test failed": 2 is an interrupted collection, 3 an internal error, 4 a usage
# error, 5 nothing collected, and a killed run gives 143. Cold review measured a SIGTERM
# being read as red in an early DRAFT — cancellation masquerading as evidence. On the first
# COMMITTED version the reproducible false PROVEN is the marker case (a failed `cd` giving
# the subshell status 1), not the signal one; see the decision record.
# `--continue-on-collection-errors` is passed so a genuine collection failure
# (what happens when the fix ADDED the module the test imports) is reported as a test
# error and exits 1 instead of vanishing into the ambiguous 2/3 band. Measured: pass 0,
# fail 1, collection-error 2 (1 with the flag), SIGTERM 143.
#
# WHAT THIS DOES NOT DO, stated because a prover that overclaims is the defect it exists
# to catch: it checks a test CAN go red, not that it asserts the RIGHT property. #776's
# `test_output_survives_a_SIGTERM` would have passed this check while still never reading
# `returncode` — the one thing its fix changed. That judgement stays with review.
#
# Usage:
#   prove-fix.sh [--repo DIR] <commit> [test-selector]
#
# The selector may be omitted when the commit carries a `Proves: <selector>` trailer.
# There is deliberately no heuristic fallback: guessing from the touched test files
# silently does nothing when a fix edits an existing test, and a prover that quietly
# proves nothing is worse than one that refuses.
#
# Exit codes:
#   0  PROVEN    — green with the fix, red (pytest exit 1) without it
#   1  UNPROVEN  — the tests passed without the fix; they do not discriminate
#   2  usage / bad arguments
#   3  no selector (no argument and no `Proves:` trailer)
#   4  nothing to revert (no non-test files) — a docs/CI-only commit; opt out explicitly
#   5  environment/git/selector failure, or a run whose exit code is not interpretable
#   6  control failed — the selector does not even pass WITH the fix

set -uo pipefail

REPO="$PWD"
if [ "${1:-}" = "--repo" ]; then
  REPO="${2:-}"; shift 2
fi
COMMIT="${1:-}"
SELECTOR="${2:-}"

TMP=""
TMP_ENUM=""
# ONE cleanup, ONE EXIT trap. An earlier version installed a SECOND `trap ... EXIT` later in
# the script to remove TMP_ENUM; `on_signal` then ran `trap - EXIT`, disarming it, so a
# signalled run leaked that directory. Folding both removals in here removes the ordering
# subtlety rather than adding a third trap to compensate for it.
cleanup() {
  if [ -n "$TMP" ]; then
    for d in "$TMP/wt-control" "$TMP/wt-reverted"; do
      [ -d "$d" ] && git -C "$REPO" worktree remove --force "$d" >/dev/null 2>&1
    done
    rm -rf "$TMP" >/dev/null 2>&1
  fi
  [ -n "$TMP_ENUM" ] && rm -rf "$TMP_ENUM" >/dev/null 2>&1
  return 0
}
# A signal must not look like a verdict: clean up, then exit non-zero EXPLICITLY. Without
# that explicit exit the handler falls through and the previous status stands, which is
# how a cancelled run printed PROVEN in the draft.
on_signal() { cleanup; trap - EXIT; printf 'prove-fix: interrupted by signal\n' >&2; exit 5; }
trap cleanup EXIT
trap on_signal INT TERM

die() { printf '%s\n' "$1" >&2; exit "$2"; }

[ -n "$COMMIT" ] || die "usage: prove-fix.sh [--repo DIR] <commit> [test-selector]" 2
git -C "$REPO" rev-parse --git-dir >/dev/null 2>&1 || die "not a git repository: $REPO" 5
command -v python3 >/dev/null 2>&1 || die "python3 is required to run the tests" 5

git -C "$REPO" rev-parse --verify --quiet "$COMMIT^{commit}" >/dev/null \
  || die "no such commit: $COMMIT" 5
SHA="$(git -C "$REPO" rev-parse "$COMMIT")"

# A merge commit has several parents, so "the code before this change" is ambiguous.
# Refuse rather than silently taking the first parent: a `Proves:` trailer on a merge is a
# claim this script cannot evaluate, and quietly evaluating a different one would be the
# overclaim it exists to prevent.
PARENTS="$(git -C "$REPO" rev-list --parents -n 1 "$SHA")" || die "rev-list failed for $SHA" 5
NPARENT=$(( $(printf '%s' "$PARENTS" | wc -w) - 1 ))
[ "$NPARENT" -le 1 ] || die \
"cannot prove a MERGE commit ($NPARENT parents): $SHA
'The code before this change' is ambiguous across parents. Put the Proves: trailer on the
commit that carries the fix." 5

PARENT="$(git -C "$REPO" rev-parse --verify --quiet "${SHA}^" || true)"
[ -n "$PARENT" ] || die "cannot prove a root commit (no parent to revert to): $SHA" 5

# ---------------------------------------------------------------- selector
if [ -z "$SELECTOR" ]; then
  RAW_TRAILERS="$(git -C "$REPO" show -s --format='%(trailers:key=Proves,valueonly)' "$SHA")"
  # More than one `Proves:` means the commit claims two proofs and only the first would be
  # checked — the rest would read as covered while never running.
  if [ "$(printf '%s\n' "$RAW_TRAILERS" | grep -c .)" -gt 1 ]; then
    die "commit carries more than one 'Proves:' trailer; only one is checked, so the others
would read as proven without ever running. Use a single selector." 3
  fi
  SELECTOR="$(printf '%s\n' "$RAW_TRAILERS" | head -1)"
  # Trim surrounding whitespace ONLY. Not `xargs`: it applies shell-ish quote parsing, so
  # a legitimate parametrised node id like `test_x[can't]` makes it report an unterminated
  # quote and yield an empty selector — silently dropping a real claim.
  SELECTOR="${SELECTOR#"${SELECTOR%%[![:space:]]*}"}"
  SELECTOR="${SELECTOR%"${SELECTOR##*[![:space:]]}"}"
fi
# Split ONCE, deliberately, into an array — then always expand as "${SEL[@]}". Leaving the
# scalar unquoted at the call site would also apply PATHNAME EXPANSION, so a node id
# containing a glob character could select different tests than the trailer names.
read -r -a SEL <<< "$SELECTOR"

[ -n "$SELECTOR" ] || die \
"no test selector: pass one, or give the commit a 'Proves: <selector>' trailer.
Refusing to guess from the touched test files — that silently proves nothing when a fix
edits an existing test, which is the failure mode this script exists to prevent." 3

# ---------------------------------------------------------------- classify
TMP_ENUM="$(mktemp -d "${TMPDIR:-/tmp}/prove-fix-enum.XXXXXX")" || die "mktemp failed" 5
# NUL-delimited --name-status: git tells us whether each path was Added, Modified,
# Deleted or Renamed. Deriving added-ness from git's own status letter beats probing the
# parent blob and interpreting the error text — `cat-file -e parent:new.py` fails with a
# non-empty message for a legitimately-added file, so "non-empty stderr means git broke"
# both mis-classifies that case and would leave real git failures indistinguishable.
# -z also protects paths containing newlines/quotes, which git otherwise quotes and which
# would then silently stay un-reverted while we claim to have reverted them.
# Capture to a file with a CHECKED status first. Inside `done < <(git ...)` the producer's
# exit status is unavailable, so a git failure that had already emitted one complete record
# would pass the non-empty check and revert only PART of the commit — overstating "without
# the fix" and manufacturing a proof from an incomplete revert.
NS="$TMP_ENUM/name-status"
git -C "$REPO" diff-tree --no-commit-id -r -z --name-status "$SHA" >"$NS" \
  || die "git diff-tree failed for $SHA — cannot enumerate what to revert" 5

CHANGED=(); STATUS=()
while IFS= read -r -d '' st; do
  IFS= read -r -d '' path || die "truncated --name-status stream after status '$st'" 5
  case "$st" in
    R*|C*) IFS= read -r -d '' newpath \
             || die "truncated rename/copy record after '$st' '$path'" 5
           CHANGED+=("$newpath"); STATUS+=("A")     # the new path is absent in the parent
           CHANGED+=("$path");    STATUS+=("D") ;;  # the old path is present in it
    *)     CHANGED+=("$path");    STATUS+=("${st:0:1}") ;;
  esac
done <"$NS"
[ "${#CHANGED[@]}" -gt 0 ] || die "commit touches no files, or enumeration failed: $SHA" 5

is_test_path() {
  case "$1" in
    scripts/tests/*|*/scripts/tests/*) return 0 ;;
    *[Tt]ests/*)                       return 0 ;;
    *.test.ts|*.test.tsx|*.test.js)    return 0 ;;
    *_test.py|test_*.py)               return 0 ;;
    *) return 1 ;;
  esac
}

NON_TEST=(); NON_TEST_ST=(); TEST_FILES=()
for i in "${!CHANGED[@]}"; do
  f="${CHANGED[$i]}"
  if is_test_path "$f"; then
    TEST_FILES+=("$f")
  else
    NON_TEST+=("$f"); NON_TEST_ST+=("${STATUS[$i]}")
  fi
done

if [ "${#NON_TEST[@]}" -eq 0 ]; then
  die "nothing to revert: $SHA changes only test files.
A commit with no code side cannot be proven this way. If it is a docs/CI-only fix, opt out
explicitly with a reason rather than letting this pass silently." 4
fi

printf 'commit    %s\n' "$SHA"
printf 'selector  %s\n' "$SELECTOR"
printf 'reverting %d non-test file(s), keeping %d test file(s)\n' \
       "${#NON_TEST[@]}" "${#TEST_FILES[@]}"

TMP="$(mktemp -d "${TMPDIR:-/tmp}/prove-fix.XXXXXX")" || die "mktemp failed" 5
mkdir -p "$TMP/tmp-control" "$TMP/tmp-reverted" || die "could not create phase temp dirs" 5

# Each phase gets its OWN worktree and TMPDIR, and pytest's cache is disabled. Sharing one
# worktree lets state written during the control run decide the second run: a test that
# creates a marker and fails when it already exists would "go red" with the fix still in
# place — a false PROVEN manufactured entirely by the harness.
#
# PYTEST_RC IS READ FROM A MARKER, NOT FROM THE SUBSHELL. `( cd X && pytest ) ; rc=$?`
# returns the SUBSHELL's status, and a failed `cd` or a failed redirection also yields 1 —
# which the "only exit 1 is red" rule would then accept as a witnessed test failure with
# pytest never having run. Cold review found exactly that. The marker file is written only
# after pytest RETURNS, so its absence means "pytest did not complete" and can never be
# mistaken for a verdict.
PYTEST_RC=""
run_phase() {  # $1 = worktree, $2 = label; sets PYTEST_RC or returns non-zero
  rm -f "$TMP/rc-$2"
  (
    cd "$1" || exit 91
    exec >"$TMP/out-$2.txt" 2>&1 || exit 92
    PYTHONPATH=. TMPDIR="$TMP/tmp-$2" PYTEST_ADDOPTS= \
      python3 -m pytest "${SEL[@]}" -q --continue-on-collection-errors -p no:cacheprovider
    printf '%s' "$?" > "$TMP/rc-$2"
  )
  if [ ! -s "$TMP/rc-$2" ]; then
    return 1   # pytest never completed — harness failure, NOT a test result
  fi
  PYTEST_RC="$(cat "$TMP/rc-$2")"
  return 0
}

# ---------------------------------------------------------------- control
git -C "$REPO" worktree add --detach "$TMP/wt-control" "$SHA" >/dev/null 2>&1 \
  || die "could not create the control worktree at $SHA" 5

echo "--- control: running the selector WITH the fix ---"
if ! run_phase "$TMP/wt-control" control; then
  sed -n '1,25p' "$TMP/out-control.txt" 2>/dev/null >&2
  die "the control run did not complete: pytest produced no exit status.
This is a harness failure (bad cd, unwritable log, missing interpreter), not a test
result, and must never be reported as a verdict." 5
fi
CONTROL_RC="$PYTEST_RC"
echo "--- control pytest exit: $CONTROL_RC ---"

if [ "$CONTROL_RC" -ne 0 ]; then
  sed -n '1,25p' "$TMP/out-control.txt" >&2
  case "$CONTROL_RC" in
    4|5) die "selector ran NO tests (pytest exit $CONTROL_RC): $SELECTOR
Exit 4 means the path does not exist; 5 means nothing was collected. Fix the selector." 5 ;;
    *)   die "control FAILED: the selector does not pass WITH the fix (pytest exit $CONTROL_RC).
A test that is already red proves nothing by being red after a revert. Fix the test or the
selector first." 6 ;;
  esac
fi

# ---------------------------------------------------------------- reverted
git -C "$REPO" worktree add --detach "$TMP/wt-reverted" "$SHA" >/dev/null 2>&1 \
  || die "could not create the reverted worktree at $SHA" 5

for i in "${!NON_TEST[@]}"; do
  f="${NON_TEST[$i]}"; st="${NON_TEST_ST[$i]}"
  case "$st" in
    A)  # added by this commit: it has no parent version, so remove it
        rm -f "$TMP/wt-reverted/$f" || die "could not remove added file $f" 5 ;;
    M|D|T)
        git -C "$TMP/wt-reverted" checkout "$PARENT" -- "$f" 2>/dev/null \
          || die "could not revert $f to $PARENT (status $st)" 5 ;;
    *)  die "unhandled git status '$st' for $f — refusing to guess how to revert it" 5 ;;
  esac
done

echo "--- running the selector WITHOUT the fix ---"
if ! run_phase "$TMP/wt-reverted" reverted; then
  sed -n '1,25p' "$TMP/out-reverted.txt" 2>/dev/null >&2
  die "the reverted run did not complete: pytest produced no exit status.
This is a harness failure, not a witnessed red. Accepting it would let a broken cd or an
unwritable log masquerade as proof — the defect this script exists to catch." 5
fi
RC="$PYTEST_RC"
# A non-numeric status makes BOTH `[ "$RC" -eq 0 ]` and `[ "$RC" -ne 1 ]` return 2, so
# control would fall through to PROVEN — a fail-OPEN default in the one place the whole
# safety argument rests. Unreachable today (the marker's only writer is `printf '%s' "$?"`),
# but every other branch here is deliberately fail-closed and this one should be too.
case "$RC" in
  ''|*[!0-9]*) die "unreadable pytest status '$RC' — refusing to guess a verdict" 5 ;;
esac
echo "--- pytest exit: $RC ---"

# ---------------------------------------------------------------- verdict
if [ "$RC" -eq 0 ]; then
  printf '\nUNPROVEN  %s\n' "$SHA"
  printf '%s\n' "The named tests PASS without the fix, so they do not discriminate against"
  printf '%s\n' "its absence. Strengthen the test until reverting the fix reddens it."
  exit 1
fi

if [ "$RC" -ne 1 ]; then
  sed -n '1,25p' "$TMP/out-reverted.txt" >&2
  die "cannot interpret pytest exit $RC as a test failure.
Only exit 1 counts as red. 2 is an interrupted collection, 3 an internal error, 4 a usage
error, 5 nothing collected, 130/143 a signal — none is evidence that the test
discriminates, and treating them as red lets a cancelled or broken run prove a fix." 5
fi

# Print WHICH tests failed. This script's own docstring says review must judge whether the
# RIGHT property failed; hiding the report would make that judgement impossible.
printf '\n--- failures observed without the fix (first 25 lines) ---\n'
sed -n '1,25p' "$TMP/out-reverted.txt"

printf '\nPROVEN    %s\n' "$SHA"
printf '%s\n' "Green with the fix (control), red without it (pytest exit 1)."
exit 0